mirror of https://github.com/TriliumNext/Notes
DB dump tool feature complete
zadam
parent
67cce5f817
commit
5481375347
@ -1,132 +1,33 @@
|
|||||||
#!/usr/bin/env node
|
#!/usr/bin/env node
|
||||||
|
|
||||||
const fs = require('fs');
|
const yargs = require('yargs/yargs')
|
||||||
const sql = require("./inc/sql");
|
const { hideBin } = require('yargs/helpers')
|
||||||
|
const dumpService = require("./inc/dump.js");
|
||||||
const args = process.argv.slice(2);
|
|
||||||
const sanitize = require('sanitize-filename');
|
yargs(hideBin(process.argv))
|
||||||
const path = require("path");
|
.command('$0 <path_to_document> <target_directory>', 'dump the contents of document.db into the target directory', (yargs) => {
|
||||||
const mimeTypes = require("mime-types");
|
return yargs
|
||||||
|
.positional('path_to_document', { describe: 'path to the document.db' })
|
||||||
if (args[0] === '-h' || args[0] === '--help') {
|
.positional('target_directory', { describe: 'path of the directory into which the notes should be dumped' })
|
||||||
printHelp();
|
}, (argv) => {
|
||||||
process.exit(0);
|
try {
|
||||||
}
|
dumpService.dumpDocument(argv.path_to_document, argv.target_directory, {
|
||||||
|
includeDeleted: argv.includeDeleted,
|
||||||
if (args.length !== 2) {
|
password: argv.password
|
||||||
console.error(`Exactly 2 arguments are expected. Run with --help to see usage.`);
|
});
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
const [documentPath, targetPath] = args;
|
|
||||||
|
|
||||||
if (!fs.existsSync(documentPath)) {
|
|
||||||
console.error(`Path to document '${documentPath}' has not been found. Run with --help to see usage.`);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!fs.existsSync(targetPath)) {
|
|
||||||
const ret = fs.mkdirSync(targetPath, { recursive: true });
|
|
||||||
|
|
||||||
if (!ret) {
|
|
||||||
console.error(`Target path '${targetPath}' could not be created. Run with --help to see usage.`);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sql.openDatabase(documentPath);
|
|
||||||
|
|
||||||
const existingPaths = {};
|
|
||||||
|
|
||||||
dumpNote(targetPath, 'root');
|
|
||||||
|
|
||||||
function getFileName(note, childTargetPath, safeTitle) {
|
|
||||||
let existingExtension = path.extname(safeTitle).toLowerCase();
|
|
||||||
let newExtension;
|
|
||||||
|
|
||||||
if (note.type === 'text') {
|
|
||||||
newExtension = 'html';
|
|
||||||
} else if (note.mime === 'application/x-javascript' || note.mime === 'text/javascript') {
|
|
||||||
newExtension = 'js';
|
|
||||||
} else if (existingExtension.length > 0) { // if the page already has an extension, then we'll just keep it
|
|
||||||
newExtension = null;
|
|
||||||
} else {
|
|
||||||
if (note.mime?.toLowerCase()?.trim() === "image/jpg") { // image/jpg is invalid but pretty common
|
|
||||||
newExtension = 'jpg';
|
|
||||||
} else {
|
|
||||||
newExtension = mimeTypes.extension(note.mime) || "dat";
|
|
||||||
}
|
}
|
||||||
}
|
catch (e) {
|
||||||
|
console.error(`Unrecoverable error:`, e);
|
||||||
let fileNameWithPath = childTargetPath;
|
process.exit(1);
|
||||||
|
|
||||||
// if the note is already named with extension (e.g. "jquery"), then it's silly to append exact same extension again
|
|
||||||
if (newExtension && existingExtension !== "." + newExtension.toLowerCase()) {
|
|
||||||
fileNameWithPath += "." + newExtension;
|
|
||||||
}
|
|
||||||
return fileNameWithPath;
|
|
||||||
}
|
|
||||||
|
|
||||||
function dumpNote(targetPath, noteId) {
|
|
||||||
console.log(`Dumping note ${noteId}`);
|
|
||||||
|
|
||||||
const note = sql.getRow("SELECT * FROM notes WHERE noteId = ?", [noteId]);
|
|
||||||
|
|
||||||
let safeTitle = sanitize(note.title);
|
|
||||||
|
|
||||||
if (safeTitle.length > 20) {
|
|
||||||
safeTitle = safeTitle.substring(0, 20);
|
|
||||||
}
|
|
||||||
|
|
||||||
let childTargetPath = targetPath + '/' + safeTitle;
|
|
||||||
|
|
||||||
for (let i = 1; i < 100000 && childTargetPath in existingPaths; i++) {
|
|
||||||
childTargetPath = targetPath + '/' + safeTitle + '_' + i;
|
|
||||||
}
|
|
||||||
|
|
||||||
existingPaths[childTargetPath] = true;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const {content} = sql.getRow("SELECT content FROM note_contents WHERE noteId = ?", [noteId]);
|
|
||||||
|
|
||||||
if (!isContentEmpty(content)) {
|
|
||||||
const fileNameWithPath = getFileName(note, childTargetPath, safeTitle);
|
|
||||||
|
|
||||||
fs.writeFileSync(fileNameWithPath, content);
|
|
||||||
}
|
}
|
||||||
}
|
})
|
||||||
catch (e) {
|
.option('password', {
|
||||||
console.log(`Writing ${note.noteId} failed with error ${e.message}`);
|
type: 'string',
|
||||||
}
|
description: 'Set password to be able to decrypt protected notes.'
|
||||||
|
})
|
||||||
const childNoteIds = sql.getColumn("SELECT noteId FROM branches WHERE parentNoteId = ?", [noteId]);
|
.option('include-deleted', {
|
||||||
|
type: 'boolean',
|
||||||
if (childNoteIds.length > 0) {
|
default: false,
|
||||||
fs.mkdirSync(childTargetPath, { recursive: true });
|
description: 'If set to true, dump also deleted notes.'
|
||||||
|
})
|
||||||
for (const childNoteId of childNoteIds) {
|
.parse();
|
||||||
dumpNote(childTargetPath, childNoteId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function isContentEmpty(content) {
|
|
||||||
if (!content) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof content === "string") {
|
|
||||||
return !content.trim() || content.trim() === '<p></p>';
|
|
||||||
}
|
|
||||||
else if (Buffer.isBuffer(content)) {
|
|
||||||
return content.length === 0;
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function printHelp() {
|
|
||||||
console.log(`Trilium Notes DB dump tool. Usage:
|
|
||||||
node dump-db.js PATH_TO_DOCUMENT_DB TARGET_PATH`);
|
|
||||||
}
|
|
||||||
|
|||||||
@ -0,0 +1,171 @@
|
|||||||
|
const fs = require("fs");
|
||||||
|
const sanitize = require("sanitize-filename");
|
||||||
|
const sql = require("./sql.js");
|
||||||
|
const decryptService = require("./decrypt.js");
|
||||||
|
const dataKeyService = require("./data_key.js");
|
||||||
|
const extensionService = require("./extension.js");
|
||||||
|
|
||||||
|
function dumpDocument(documentPath, targetPath, options) {
|
||||||
|
const stats = {
|
||||||
|
succeeded: 0,
|
||||||
|
failed: 0,
|
||||||
|
protected: 0,
|
||||||
|
deleted: 0
|
||||||
|
};
|
||||||
|
|
||||||
|
validatePaths(documentPath, targetPath);
|
||||||
|
|
||||||
|
sql.openDatabase(documentPath);
|
||||||
|
|
||||||
|
const dataKey = dataKeyService.getDataKey(options.password);
|
||||||
|
|
||||||
|
const existingPaths = {};
|
||||||
|
const noteIdToPath = {};
|
||||||
|
|
||||||
|
dumpNote(targetPath, 'root');
|
||||||
|
|
||||||
|
printDumpResults(stats, options);
|
||||||
|
|
||||||
|
function dumpNote(targetPath, noteId) {
|
||||||
|
console.log(`Reading note '${noteId}'`);
|
||||||
|
|
||||||
|
let childTargetPath, note, fileNameWithPath;
|
||||||
|
|
||||||
|
try {
|
||||||
|
note = sql.getRow("SELECT * FROM notes WHERE noteId = ?", [noteId]);
|
||||||
|
|
||||||
|
if (note.isDeleted) {
|
||||||
|
stats.deleted++;
|
||||||
|
|
||||||
|
if (!options.includeDeleted) {
|
||||||
|
console.log(`Note '${noteId}' is deleted and --include-deleted option is not used, skipping.`);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (note.isProtected) {
|
||||||
|
stats.protected++;
|
||||||
|
|
||||||
|
note.title = decryptService.decryptString(dataKey, note.title);
|
||||||
|
}
|
||||||
|
|
||||||
|
let safeTitle = sanitize(note.title);
|
||||||
|
|
||||||
|
if (safeTitle.length > 20) {
|
||||||
|
safeTitle = safeTitle.substring(0, 20);
|
||||||
|
}
|
||||||
|
|
||||||
|
childTargetPath = targetPath + '/' + safeTitle;
|
||||||
|
|
||||||
|
for (let i = 1; i < 100000 && childTargetPath in existingPaths; i++) {
|
||||||
|
childTargetPath = targetPath + '/' + safeTitle + '_' + i;
|
||||||
|
}
|
||||||
|
|
||||||
|
existingPaths[childTargetPath] = true;
|
||||||
|
|
||||||
|
if (note.noteId in noteIdToPath) {
|
||||||
|
const message = `Note '${noteId}' has been already dumped to ${noteIdToPath[note.noteId]}`;
|
||||||
|
|
||||||
|
console.log(message);
|
||||||
|
|
||||||
|
fs.writeFileSync(childTargetPath, message);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let {content} = sql.getRow("SELECT content FROM note_contents WHERE noteId = ?", [noteId]);
|
||||||
|
|
||||||
|
if (content !== null && note.isProtected && dataKey) {
|
||||||
|
content = decryptService.decrypt(dataKey, content);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isContentEmpty(content)) {
|
||||||
|
console.log(`Note '${noteId}' is empty, skipping.`);
|
||||||
|
} else {
|
||||||
|
fileNameWithPath = extensionService.getFileName(note, childTargetPath, safeTitle);
|
||||||
|
|
||||||
|
fs.writeFileSync(fileNameWithPath, content);
|
||||||
|
|
||||||
|
stats.succeeded++;
|
||||||
|
|
||||||
|
console.log(`Dumped note '${noteId}' into ${fileNameWithPath} successfully.`);
|
||||||
|
}
|
||||||
|
|
||||||
|
noteIdToPath[noteId] = childTargetPath;
|
||||||
|
}
|
||||||
|
catch (e) {
|
||||||
|
console.error(`DUMPERROR: Writing '${noteId}' failed with error '${e.message}':\n${e.stack}`);
|
||||||
|
|
||||||
|
stats.failed++;
|
||||||
|
}
|
||||||
|
|
||||||
|
const childNoteIds = sql.getColumn("SELECT noteId FROM branches WHERE parentNoteId = ?", [noteId]);
|
||||||
|
|
||||||
|
if (childNoteIds.length > 0) {
|
||||||
|
if (childTargetPath === fileNameWithPath) {
|
||||||
|
childTargetPath += '_dir';
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
fs.mkdirSync(childTargetPath, {recursive: true});
|
||||||
|
}
|
||||||
|
catch (e) {
|
||||||
|
console.error(`DUMPERROR: Creating directory ${childTargetPath} failed with error '${e.message}'`);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const childNoteId of childNoteIds) {
|
||||||
|
dumpNote(childTargetPath, childNoteId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function printDumpResults(stats, options) {
|
||||||
|
console.log('\n----------------------- STATS -----------------------');
|
||||||
|
console.log('Successfully dumpted notes: ', stats.succeeded.toString().padStart(5, ' '));
|
||||||
|
console.log('Protected notes: ', stats.protected.toString().padStart(5, ' '), options.password ? '' : '(skipped)');
|
||||||
|
console.log('Failed notes: ', stats.failed.toString().padStart(5, ' '));
|
||||||
|
console.log('Deleted notes: ', stats.deleted.toString().padStart(5, ' '), options.includeDeleted ? "(dumped)" : "(at least, skipped)");
|
||||||
|
console.log('-----------------------------------------------------');
|
||||||
|
|
||||||
|
if (!options.password && stats.protected > 0) {
|
||||||
|
console.log("\nWARNING: protected notes are present in the document but no password has been provided. Protected notes have not been dumped.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isContentEmpty(content) {
|
||||||
|
if (!content) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof content === "string") {
|
||||||
|
return !content.trim() || content.trim() === '<p></p>';
|
||||||
|
}
|
||||||
|
else if (Buffer.isBuffer(content)) {
|
||||||
|
return content.length === 0;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function validatePaths(documentPath, targetPath) {
|
||||||
|
if (!fs.existsSync(documentPath)) {
|
||||||
|
console.error(`Path to document '${documentPath}' has not been found. Run with --help to see usage.`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!fs.existsSync(targetPath)) {
|
||||||
|
const ret = fs.mkdirSync(targetPath, {recursive: true});
|
||||||
|
|
||||||
|
if (!ret) {
|
||||||
|
console.error(`Target path '${targetPath}' could not be created. Run with --help to see usage.`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
dumpDocument
|
||||||
|
};
|
||||||
@ -0,0 +1,34 @@
|
|||||||
|
const path = require("path");
|
||||||
|
const mimeTypes = require("mime-types");
|
||||||
|
|
||||||
|
function getFileName(note, childTargetPath, safeTitle) {
|
||||||
|
let existingExtension = path.extname(safeTitle).toLowerCase();
|
||||||
|
let newExtension;
|
||||||
|
|
||||||
|
if (note.type === 'text') {
|
||||||
|
newExtension = 'html';
|
||||||
|
} else if (note.mime === 'application/x-javascript' || note.mime === 'text/javascript') {
|
||||||
|
newExtension = 'js';
|
||||||
|
} else if (existingExtension.length > 0) { // if the page already has an extension, then we'll just keep it
|
||||||
|
newExtension = null;
|
||||||
|
} else {
|
||||||
|
if (note.mime?.toLowerCase()?.trim() === "image/jpg") { // image/jpg is invalid but pretty common
|
||||||
|
newExtension = 'jpg';
|
||||||
|
} else {
|
||||||
|
newExtension = mimeTypes.extension(note.mime) || "dat";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let fileNameWithPath = childTargetPath;
|
||||||
|
|
||||||
|
// if the note is already named with extension (e.g. "jquery"), then it's silly to append exact same extension again
|
||||||
|
if (newExtension && existingExtension !== "." + newExtension.toLowerCase()) {
|
||||||
|
fileNameWithPath += "." + newExtension;
|
||||||
|
}
|
||||||
|
|
||||||
|
return fileNameWithPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
getFileName
|
||||||
|
};
|
||||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue