many changes related to #1192:

- use CSS contain wherever possible to reduce subtrees of forced reflows
- reduced dependency between note and note_contents updates which will reduce number of updates to components
- optimization of "many rows" querying

db/migrations/0165__move_contentLength.sql

@@ -0,0 +1,55 @@
+CREATE TABLE IF NOT EXISTS "notes_mig" (
+                                       `noteId`	TEXT NOT NULL,
+                                       `title`	TEXT NOT NULL DEFAULT "note",
+                                       `isProtected`	INT NOT NULL DEFAULT 0,
+                                       `type` TEXT NOT NULL DEFAULT 'text',
+                                       `mime` TEXT NOT NULL DEFAULT 'text/html',
+                                       `hash` TEXT DEFAULT "" NOT NULL,
+                                       `isDeleted`	INT NOT NULL DEFAULT 0,
+                                       `deleteId`   TEXT DEFAULT NULL,
+                                       `isErased`	INT NOT NULL DEFAULT 0,
+                                       `dateCreated`	TEXT NOT NULL,
+                                       `dateModified`	TEXT NOT NULL,
+                                       `utcDateCreated`	TEXT NOT NULL,
+                                       `utcDateModified`	TEXT NOT NULL,
+                                       PRIMARY KEY(`noteId`));
+
+INSERT INTO notes_mig (noteId, title, isProtected, type, mime, hash, isDeleted, deleteId, isErased, dateCreated, dateModified, utcDateCreated, utcDateModified)
+    SELECT noteId, title, isProtected, type, mime, hash, isDeleted, deleteId, isErased, dateCreated, dateModified, utcDateCreated, utcDateModified FROM notes;
+
+DROP TABLE notes;
+ALTER TABLE notes_mig RENAME TO notes;
+
+CREATE INDEX `IDX_notes_isDeleted` ON `notes` (`isDeleted`);
+CREATE INDEX `IDX_notes_title` ON `notes` (`title`);
+CREATE INDEX `IDX_notes_type` ON `notes` (`type`);
+CREATE INDEX `IDX_notes_dateCreated` ON `notes` (`dateCreated`);
+CREATE INDEX `IDX_notes_dateModified` ON `notes` (`dateModified`);
+CREATE INDEX `IDX_notes_utcDateModified` ON `notes` (`utcDateModified`);
+CREATE INDEX `IDX_notes_utcDateCreated` ON `notes` (`utcDateCreated`);
+
+CREATE TABLE IF NOT EXISTS "note_revisions_mig" (`noteRevisionId`	TEXT NOT NULL PRIMARY KEY,
+                                             `noteId`	TEXT NOT NULL,
+                                             `title`	TEXT,
+                                             `isErased`	INT NOT NULL DEFAULT 0,
+                                             `isProtected`	INT NOT NULL DEFAULT 0,
+                                             `utcDateLastEdited` TEXT NOT NULL,
+                                             `utcDateCreated` TEXT NOT NULL,
+                                             `utcDateModified` TEXT NOT NULL,
+                                             `dateLastEdited` TEXT NOT NULL,
+                                             `dateCreated` TEXT NOT NULL,
+                                             type TEXT DEFAULT '' NOT NULL,
+                                             mime TEXT DEFAULT '' NOT NULL,
+                                             hash TEXT DEFAULT '' NOT NULL);
+
+INSERT INTO note_revisions_mig (noteRevisionId, noteId, title, isErased, isProtected, utcDateLastEdited, utcDateCreated, utcDateModified, dateLastEdited, dateCreated, type, mime, hash)
+SELECT noteRevisionId, noteId, title, isErased, isProtected, utcDateLastEdited, utcDateCreated, utcDateModified, dateLastEdited, dateCreated, type, mime, hash FROM note_revisions;
+
+DROP TABLE note_revisions;
+ALTER TABLE note_revisions_mig RENAME TO note_revisions;
+
+CREATE INDEX `IDX_note_revisions_noteId` ON `note_revisions` (`noteId`);
+CREATE INDEX `IDX_note_revisions_utcDateCreated` ON `note_revisions` (`utcDateCreated`);
+CREATE INDEX `IDX_note_revisions_utcDateLastEdited` ON `note_revisions` (`utcDateLastEdited`);
+CREATE INDEX `IDX_note_revisions_dateCreated` ON `note_revisions` (`dateCreated`);
+CREATE INDEX `IDX_note_revisions_dateLastEdited` ON `note_revisions` (`dateLastEdited`);

db/migrations/0166__add_dateModified_to_note_content.sql

@@ -0,0 +1,20 @@
+CREATE TABLE IF NOT EXISTS "note_contents_mig" (
+                                               `noteId`	TEXT NOT NULL,
+                                               `content`	TEXT NULL DEFAULT NULL,
+                                               `hash` TEXT DEFAULT "" NOT NULL,
+                                               `dateModified` TEXT NOT NULL,
+                                               `utcDateModified` TEXT NOT NULL,
+                                               PRIMARY KEY(`noteId`)
+);
+
+INSERT INTO note_contents_mig (noteId, content, hash, dateModified, utcDateModified)
+    SELECT noteId,
+           content,
+           hash,
+           (SELECT dateModified FROM notes WHERE noteId = note_contents.noteId),
+           utcDateModified
+    FROM note_contents;
+
+DROP TABLE note_contents;
+
+ALTER TABLE note_contents_mig RENAME TO note_contents;

src/entities/entity.js

@@ -26,15 +26,8 @@ class Entity {
         const origHash = this.hash;
 
         this.hash = this.generateHash();
-
-        if (this.forcedChange) {
-            this.isChanged = true;
-            delete this.forcedChange;
-        }
-        else {
         this.isChanged = origHash !== this.hash;
     }
-    }
 
     generateIdIfNecessary() {
         if (!this[this.constructor.primaryKeyName]) {

src/entities/note.js

@@ -20,7 +20,6 @@ const RELATION_DEFINITION = 'relation-definition';
  * @property {string} type - one of "text", "code", "file" or "render"
  * @property {string} mime - MIME type, e.g. "text/html"
  * @property {string} title - note title
- * @property {int} contentLength - length of content
  * @property {boolean} isProtected - true if note is protected
  * @property {boolean} isDeleted - true if note is deleted
  * @property {string|null} deleteId - ID identifying delete transaction
@@ -106,6 +105,16 @@ class Note extends Entity {
         }
     }
 
+    getContentMetadata() {
+        return sql.getRow(`
+            SELECT 
+                LENGTH(content) AS contentLength, 
+                dateModified, 
+                utcDateModified 
+            FROM note_contents 
+            WHERE noteId = ?`, [this.noteId]);
+    }
+
     /** @returns {*} */
     getJsonContent() {
         const content = this.getContent();
@@ -129,16 +138,12 @@ class Note extends Entity {
             content = Buffer.isBuffer(content) ? content : Buffer.from(content);
         }
 
-        // force updating note itself so that dateModified is represented correctly even for the content
-        this.forcedChange = true;
-        this.contentLength = content.byteLength;
-        this.save();
-
         this.content = content;
 
         const pojo = {
             noteId: this.noteId,
             content: content,
+            dateModified: dateUtils.localNowDateTime(),
             utcDateModified: dateUtils.utcNowDateTime(),
             hash: utils.hash(this.noteId + "|" + content.toString())
         };
@@ -903,10 +908,6 @@ class Note extends Entity {
             this.utcDateCreated = dateUtils.utcNowDateTime();
         }
 
-        if (this.contentLength === undefined) {
-            this.contentLength = -1;
-        }
-
         super.beforeSaving();
 
         if (this.isChanged) {

src/entities/note_revision.js

@@ -15,7 +15,6 @@ const entityChangesService = require('../services/entity_changes.js');
  * @property {string} type
  * @property {string} mime
  * @property {string} title
- * @property {int} contentLength
  * @property {boolean} isErased
  * @property {boolean} isProtected
  * @property {string} dateLastEdited
@@ -29,7 +28,7 @@ const entityChangesService = require('../services/entity_changes.js');
 class NoteRevision extends Entity {
     static get entityName() { return "note_revisions"; }
     static get primaryKeyName() { return "noteRevisionId"; }
-    static get hashedProperties() { return ["noteRevisionId", "noteId", "title", "contentLength", "isErased", "isProtected", "dateLastEdited", "dateCreated", "utcDateLastEdited", "utcDateCreated", "utcDateModified"]; }
+    static get hashedProperties() { return ["noteRevisionId", "noteId", "title", "isErased", "isProtected", "dateLastEdited", "dateCreated", "utcDateLastEdited", "utcDateCreated", "utcDateModified"]; }
 
     constructor(row) {
         super(row);
@@ -101,11 +100,6 @@ class NoteRevision extends Entity {
     }
 
     setContent(content) {
-        // force updating note itself so that utcDateModified is represented correctly even for the content
-        this.forcedChange = true;
-        this.contentLength = content === null ? 0 : content.length;
-        this.save();
-
         this.content = content;
 
         const pojo = {

src/public/app/entities/note_complement.js

@@ -6,9 +6,14 @@ class NoteComplement {
         /** @param {string} */
         this.noteId = row.noteId;
 
-        /** @param {string} */
+        /**
+         * @param {string} - can either contain the whole content (in e.g. string notes), only part (large text notes) or nothing at all (binary notes, images)
+         */
         this.content = row.content;
 
+        /** @param {int} */
+        this.contentLength = row.contentLength;
+
         /** @param {string} */
         this.dateCreated = row.dateCreated;
 
@@ -20,6 +25,12 @@ class NoteComplement {
 
         /** @param {string} */
         this.utcDateModified = row.utcDateModified;
+
+        /** @param {string} */
+        this.combinedDateModified = row.combinedDateModified;
+
+        /** @param {string} */
+        this.combinedUtcDateModified = row.combinedUtcDateModified;
     }
 }
 

src/public/app/entities/note_short.js

@@ -45,8 +45,6 @@ class NoteShort {
         this.noteId = row.noteId;
         /** @param {string} */
         this.title = row.title;
-        /** @param {int} */
-        this.contentLength = row.contentLength;
         /** @param {boolean} */
         this.isProtected = !!row.isProtected;
         /** @param {string} one of 'text', 'code', 'file' or 'render' */

src/public/app/layouts/desktop_main_window_layout.js

@@ -110,6 +110,7 @@ export default class DesktopMainWindowLayout {
             .id('root-widget')
             .css('height', '100vh')
             .child(new FlexContainer('row')
+                .css('height', '35px')
                 .child(new GlobalMenuWidget())
                 .child(new TabRowWidget())
                 .child(new TitleBarButtonsWidget()))
@@ -130,6 +131,7 @@ export default class DesktopMainWindowLayout {
                 .child(new FlexContainer('column').id('center-pane')
                     .child(new FlexContainer('row').class('title-row')
                         .cssBlock('.title-row > * { margin: 5px; }')
+                        .css('height', '55px')
                         .child(new NoteTitleWidget())
                         .child(new RunScriptButtonsWidget().hideInZenMode())
                         .child(new NoteTypeWidget().hideInZenMode())

src/public/app/services/load_results.js

@@ -120,4 +120,11 @@ export default class LoadResults {
             && this.contentNoteIdToSourceId.length === 0
             && this.options.length === 0;
     }
+
+    isEmptyForTree() {
+        return Object.keys(this.noteIdToSourceId).length === 0
+            && this.branches.length === 0
+            && this.attributes.length === 0
+            && this.noteReorderings.length === 0;
+    }
 }

src/public/app/widgets/attribute_editor.js

@@ -183,6 +183,7 @@ export default class AttributeEditorWidget extends TabAwareWidget {
 
     doRender() {
         this.$widget = $(TPL);
+        this.contentSized();
         this.$editor = this.$widget.find('.attribute-list-editor');
 
         this.initialized = this.initEditor();

src/public/app/widgets/attribute_list.js

@@ -116,6 +116,7 @@ export default class AttributeListWidget extends TabAwareWidget {
 
     doRender() {
         this.$widget = $(TPL);
+        this.contentSized();
 
         this.$promotedExpander = this.$widget.find('.attr-promoted-expander');
         this.$allAttrWrapper = this.$widget.find('.all-attr-wrapper');

src/public/app/widgets/basic_widget.js

@@ -25,6 +25,11 @@ class BasicWidget extends Component {
         return this;
     }
 
+    contentSized() {
+        this.css('contain', 'layout paint');
+        return this;
+    }
+
     collapsible() {
         this.css('min-height', '0');
         return this;

src/public/app/widgets/collapsible_widget.js

@@ -29,6 +29,7 @@ export default class CollapsibleWidget extends TabAwareWidget {
 
     doRender() {
         this.$widget = $(WIDGET_TPL);
+        this.contentSized();
         this.$widget.find('[data-target]').attr('data-target', "#" + this.componentId);
 
         this.$bodyWrapper = this.$widget.find('.body-wrapper');

src/public/app/widgets/collapsible_widgets/note_info.js

@@ -64,8 +64,8 @@ export default class NoteInfoWidget extends CollapsibleWidget {
             .attr("title", noteComplement.dateCreated);
 
         this.$dateModified
-            .text(noteComplement.dateModified.substr(0, 16))
+            .text(noteComplement.combinedDateModified.substr(0, 16))
-            .attr("title", noteComplement.dateCreated);
+            .attr("title", noteComplement.combinedDateModified);
 
         this.$type.text(note.type);
 
@@ -78,7 +78,7 @@ export default class NoteInfoWidget extends CollapsibleWidget {
     }
 
     entitiesReloadedEvent({loadResults}) {
-        if (loadResults.isNoteReloaded(this.noteId)) {
+        if (loadResults.isNoteReloaded(this.noteId) || loadResults.isNoteContentReloaded(this.noteId)) {
             this.refresh();
         }
     }

src/public/app/widgets/global_buttons.js

@@ -32,7 +32,8 @@ const WIDGET_TPL = `
 
 class GlobalButtonsWidget extends BasicWidget {
     doRender() {
-        return this.$widget = $(WIDGET_TPL);
+        this.$widget = $(WIDGET_TPL);
+        this.contentSized();
     }
 }
 

src/public/app/widgets/global_menu.js

@@ -104,6 +104,7 @@ const TPL = `
 export default class GlobalMenuWidget extends BasicWidget {
     doRender() {
         this.$widget = $(TPL);
+        this.contentSized();
 
         this.$widget.find(".show-about-dialog-button").on('click',
             () => import("../dialogs/about.js").then(d => d.showDialog()));

src/public/app/widgets/history_navigation.js

@@ -23,6 +23,7 @@ export default class HistoryNavigationWidget extends BasicWidget {
     doRender() {
         if (utils.isElectron()) {
             this.$widget = $(TPL);
+            this.contentSized();
 
             const contextMenuHandler = e => {
                 e.preventDefault();

src/public/app/widgets/note_actions.js

@@ -93,6 +93,7 @@ const TPL = `
 export default class NoteActionsWidget extends TabAwareWidget {
     doRender() {
         this.$widget = $(TPL);
+        this.contentSized();
 
         this.$showSourceButton = this.$widget.find('.show-source-button');
 

src/public/app/widgets/note_paths.js

@@ -50,6 +50,7 @@ const TPL = `
 export default class NotePathsWidget extends TabAwareWidget {
     doRender() {
         this.$widget = $(TPL);
+        this.contentSized();
 
         this.$currentPath = this.$widget.find('.current-path');
         this.$dropdown = this.$widget.find(".dropdown");

src/public/app/widgets/note_tree.js

@@ -957,6 +957,10 @@ export default class NoteTreeWidget extends TabAwareWidget {
     }
 
     async entitiesReloadedEvent({loadResults}) {
+        if (loadResults.isEmptyForTree()) {
+            return;
+        }
+
         const activeNode = this.getActiveNode();
         const activeNodeFocused = activeNode && activeNode.hasFocus();
         const nextNode = activeNode ? (activeNode.getNextSibling() || activeNode.getPrevSibling() || activeNode.getParent()) : null;

src/public/app/widgets/note_type.js

@@ -34,6 +34,7 @@ const TPL = `
 export default class NoteTypeWidget extends TabAwareWidget {
     doRender() {
         this.$widget = $(TPL);
+        this.contentSized();
 
         this.$widget.on('show.bs.dropdown', () => this.renderDropdown());
 

src/public/app/widgets/side_pane_toggles.js

@@ -43,6 +43,8 @@ export default class SidePaneToggles extends BasicWidget {
 
         this.$widget.find(".show-left-pane-button").on('click', () => this.toggleAndSave('left', true));
         this.$widget.find(".hide-left-pane-button").on('click', () => this.toggleAndSave('left', false));
+
+        this.$widget.css("contain", "none"); // this widget overflows so we need to override default containment
     }
 
     toggleSidebar(side, show) {

src/public/app/widgets/standard_top_widget.js

@@ -10,6 +10,7 @@ const TPL = `
         display: flex;
         align-items: center;
         padding-top: 4px;
+        height: 35px;
     }
     
     .standard-top-widget button {

src/public/app/widgets/title_bar_buttons.js

@@ -35,6 +35,7 @@ export default class TitleBarButtonsWidget extends BasicWidget {
         }
 
         this.$widget = $(TPL);
+        this.contentSized();
 
         const $minimizeBtn = this.$widget.find(".minimize-btn");
         const $maximizeBtn = this.$widget.find(".maximize-btn");

src/public/app/widgets/type_widgets/file.js

@@ -126,11 +126,11 @@ export default class FileTypeWidget extends TypeWidget {
 
         this.$fileNoteId.text(note.noteId);
         this.$fileName.text(attributeMap.originalFileName || "?");
-        this.$fileSize.text(note.contentLength + " bytes");
         this.$fileType.text(note.mime);
 
         const noteComplement = await this.tabContext.getNoteComplement();
 
+        this.$fileSize.text(noteComplement.contentLength + " bytes");
         this.$previewContent.empty().hide();
         this.$pdfPreview.attr('src', '').empty().hide();
 

src/public/app/widgets/type_widgets/image.js

@@ -127,8 +127,10 @@ class ImageTypeWidget extends TypeWidget {
 
         this.$widget.show();
 
+        const noteComplement = await this.tabContext.getNoteComplement();
+
         this.$fileName.text(attributeMap.originalFileName || "?");
-        this.$fileSize.text(note.contentLength + " bytes");
+        this.$fileSize.text(noteComplement.contentLength + " bytes");
         this.$fileType.text(note.mime);
 
         const imageHash = utils.randomString(10);

src/public/stylesheets/style.css

@@ -635,6 +635,10 @@ a.external:not(.no-arrow):after, a[href^="http://"]:not(.no-arrow):after, a[href
     pointer-events: none;
 }
 
+.component {
+    contain: strict;
+}
+
 .toast {
     background-color: var(--accented-background-color) !important;
     color: var(--main-text-color) !important;

src/routes/api/note_revisions.js

@@ -9,8 +9,12 @@ const path = require('path');
 
 function getNoteRevisions(req) {
     return repository.getEntities(`
-        SELECT * FROM note_revisions 
+        SELECT note_revisions.*,
-        WHERE noteId = ? AND isErased = 0
+               LENGTH(note_revision_contents.content) AS contentLength
+        FROM note_revisions
+        JOIN note_revision_contents ON note_revisions.noteRevisionId = note_revision_contents.noteRevisionId 
+        WHERE noteId = ? 
+          AND isErased = 0
         ORDER BY utcDateCreated DESC`, [req.params.noteId]);
 }
 

src/routes/api/notes.js

@@ -23,6 +23,11 @@ function getNote(req) {
         }
     }
 
+    const contentMetadata = note.getContentMetadata();
+
+    note.combinedUtcDateModified = note.utcDateModified > contentMetadata.utcDateModified ? note.utcDateModified : contentMetadata.utcDateModified;
+    note.combinedDateModified = note.utcDateModified > contentMetadata.utcDateModified ? note.dateModified : contentMetadata.dateModified;
+
     return note;
 }
 

src/services/app_info.js

@@ -4,8 +4,8 @@ const build = require('./build');
 const packageJson = require('../../package');
 const {TRILIUM_DATA_DIR} = require('./data_dir');
 
-const APP_DB_VERSION = 164;
+const APP_DB_VERSION = 166;
-const SYNC_VERSION = 15;
+const SYNC_VERSION = 16;
 const CLIPPER_PROTOCOL_VERSION = "1.0";
 
 module.exports = {

src/services/note_cache/entities/note.js

@@ -14,8 +14,6 @@ class Note {
         this.type = row.type;
         /** @param {string} */
         this.mime = row.mime;
-        /** @param {number} */
-        this.contentLength = row.contentLength;
         /** @param {string} */
         this.dateCreated = row.dateCreated;
         /** @param {string} */
@@ -182,8 +180,6 @@ class Note {
             }
 
             this.flatTextCache = this.flatTextCache.toLowerCase();
-
-            console.log(this.flatTextCache);
         }
 
         return this.flatTextCache;

src/services/note_cache/note_cache_loader.js

@@ -15,7 +15,7 @@ sqlInit.dbReady.then(() => {
 function load() {
     noteCache.reset();
 
-    for (const row of sql.iterateRows(`SELECT noteId, title, type, mime, isProtected, dateCreated, dateModified, utcDateCreated, utcDateModified, contentLength FROM notes WHERE isDeleted = 0`, [])) {
+    for (const row of sql.iterateRows(`SELECT noteId, title, type, mime, isProtected, dateCreated, dateModified, utcDateCreated, utcDateModified FROM notes WHERE isDeleted = 0`, [])) {
         new Note(noteCache, row);
     }
 

src/services/note_revisions.js

@@ -34,7 +34,6 @@ function createNoteRevision(note) {
         noteId: note.noteId,
         // title and text should be decrypted now
         title: note.title,
-        contentLength: -1, // will be updated in .setContent()
         type: note.type,
         mime: note.mime,
         isProtected: false, // will be fixed in the protectNoteRevisions() call

src/services/notes.js

@@ -466,7 +466,6 @@ function saveNoteRevision(note) {
             noteId: note.noteId,
             // title and text should be decrypted now
             title: note.title,
-            contentLength: -1, // will be updated in .setContent()
             type: note.type,
             mime: note.mime,
             isProtected: false, // will be fixed in the protectNoteRevisions() call
@@ -699,7 +698,6 @@ function eraseDeletedNotes() {
     sql.executeMany(`
         UPDATE notes 
         SET title = '[deleted]',
-            contentLength = 0,
             isProtected = 0,
             isErased = 1
         WHERE noteId IN (???)`, noteIdsToErase);
@@ -719,8 +717,7 @@ function eraseDeletedNotes() {
     sql.executeMany(`
         UPDATE note_revisions 
         SET isErased = 1,
-            title = NULL,
+            title = NULL
-            contentLength = 0
         WHERE isErased = 0 AND noteId IN (???)`, noteIdsToErase);
 
     sql.executeMany(`

src/services/search/expressions/property_comparison.js

@@ -18,7 +18,6 @@ const PROP_MAPPING = {
     "datemodified": "dateModified",
     "utcdatecreated": "utcDateCreated",
     "utcdatemodified": "utcDateModified",
-    "contentlength": "contentLength",
     "parentcount": "parentCount",
     "childrencount": "childrenCount",
     "attributecount": "attributeCount",

src/services/search/value_extractor.js

@@ -15,7 +15,6 @@ const PROP_MAPPING = {
     "datemodified": "dateModified",
     "utcdatecreated": "utcDateCreated",
     "utcdatemodified": "utcDateModified",
-    "contentlength": "contentLength",
     "parentcount": "parentCount",
     "childrencount": "childrenCount",
     "attributecount": "attributeCount",

src/services/sql.js

@@ -93,9 +93,9 @@ function getValue(query, params = []) {
     return row[Object.keys(row)[0]];
 }
 
-const PARAM_LIMIT = 900; // actual limit is 999
+// smaller values can result in better performance due to better usage of statement cache
+const PARAM_LIMIT = 100;
 
-// this is to overcome 999 limit of number of query parameters
 function getManyRows(query, params) {
     let results = [];
 
@@ -114,7 +114,11 @@ function getManyRows(query, params) {
         const questionMarks = curParams.map(() => ":param" + i++).join(",");
         const curQuery = query.replace(/\?\?\?/g, questionMarks);
 
-        const subResults = dbConnection.prepare(curQuery).all(curParamsObj);
+        const statement = curParams.length === PARAM_LIMIT
+            ? stmt(curQuery)
+            : dbConnection.prepare(curQuery);
+
+        const subResults = statement.all(curParamsObj);
         results = results.concat(subResults);
     }
 

src/services/tree.js

@@ -12,7 +12,6 @@ function getNotes(noteIds) {
         SELECT 
           noteId,
           title,
-          contentLength,
           isProtected,
           type,
           mime,