sync fixes

src/services/consistency_checks.js

@@ -758,7 +758,7 @@ class ConsistencyChecks {
             return `${tableName}: ${count}`;
         }
 
-        const tables = [ "notes", "revisions", "attachments", "branches", "attributes", "etapi_tokens" ];
+        const tables = [ "notes", "revisions", "attachments", "branches", "attributes", "etapi_tokens", "blobs" ];
 
         log.info(`Table counts: ${tables.map(tableName => getTableRowCount(tableName)).join(", ")}`);
     }
@@ -767,7 +767,13 @@ class ConsistencyChecks {
         let elapsedTimeMs;
 
         await syncMutexService.doExclusively(() => {
-            elapsedTimeMs = this.runChecksInner();
+            const startTimeMs = Date.now();
+
+            this.runDbDiagnostics();
+
+            this.runAllChecksAndFixers();
+
+            elapsedTimeMs = Date.now() - startTimeMs;
         });
 
         if (this.unrecoveredConsistencyErrors) {
@@ -781,16 +787,6 @@ class ConsistencyChecks {
             );
         }
     }
-
-    runChecksInner() {
-        const startTimeMs = Date.now();
-
-        this.runDbDiagnostics();
-
-        this.runAllChecksAndFixers();
-
-        return Date.now() - startTimeMs;
-    }
 }
 
 function getBlankContent(isProtected, type, mime) {
@@ -825,11 +821,6 @@ async function runOnDemandChecks(autoFix) {
     await consistencyChecks.runChecks();
 }
 
-function runOnDemandChecksWithoutExclusiveLock(autoFix) {
-    const consistencyChecks = new ConsistencyChecks(autoFix);
-    consistencyChecks.runChecksInner();
-}
-
 function runEntityChangesChecks() {
     const consistencyChecks = new ConsistencyChecks(true);
     consistencyChecks.findEntityChangeIssues();
@@ -844,6 +835,5 @@ sqlInit.dbReady.then(() => {
 
 module.exports = {
     runOnDemandChecks,
-    runOnDemandChecksWithoutExclusiveLock,
     runEntityChangesChecks
 };

src/services/entity_changes.js

@@ -15,6 +15,12 @@ function putEntityChangeWithInstanceId(origEntityChange, instanceId) {
     putEntityChange(ec);
 }
 
+function putEntityChangeWithForcedChange(origEntityChange) {
+    const ec = {...origEntityChange, changeId: null};
+
+    putEntityChange(ec);
+}
+
 function putEntityChange(origEntityChange) {
     const ec = {...origEntityChange};
 
@@ -66,13 +72,37 @@ function putEntityChangeForOtherInstances(ec) {
 function addEntityChangesForSector(entityName, sector) {
     const entityChanges = sql.getRows(`SELECT * FROM entity_changes WHERE entityName = ? AND SUBSTR(entityId, 1, 1) = ?`, [entityName, sector]);
 
+    let entitiesInserted = entityChanges.length;
+
     sql.transactional(() => {
+        if (entityName === 'blobs') {
+            entitiesInserted += addEntityChangesForDependingEntity(sector, 'notes', 'noteId');
+            entitiesInserted += addEntityChangesForDependingEntity(sector, 'attachments', 'attachmentId');
+            entitiesInserted += addEntityChangesForDependingEntity(sector, 'revisions', 'revisionId');
+        }
+
         for (const ec of entityChanges) {
-            putEntityChange(ec);
+            putEntityChangeWithForcedChange(ec);
         }
     });
 
-    log.info(`Added sector ${sector} of '${entityName}' (${entityChanges.length} entities) to the sync queue.`);
+    log.info(`Added sector ${sector} of '${entityName}' (${entitiesInserted} entities) to the sync queue.`);
+}
+
+function addEntityChangesForDependingEntity(sector, tableName, primaryKeyColumn) {
+    // problem in blobs might be caused by problem in entity referencing the blob
+    const dependingEntityChanges = sql.getRows(`
+                SELECT dep_change.* 
+                FROM entity_changes orig_sector
+                JOIN ${tableName} ON ${tableName}.blobId = orig_sector.entityId
+                JOIN entity_changes dep_change ON dep_change.entityName = '${tableName}' AND dep_change.entityId = ${tableName}.${primaryKeyColumn}
+                WHERE orig_sector.entityName = 'blobs' AND SUBSTR(orig_sector.entityId, 1, 1) = ?`, [sector]);
+
+    for (const ec of dependingEntityChanges) {
+        putEntityChangeWithForcedChange(ec);
+    }
+
+    return dependingEntityChanges.length;
 }
 
 function cleanupEntityChangesForMissingEntities(entityName, entityPrimaryKey) {
@@ -161,6 +191,7 @@ function recalculateMaxEntityChangeId() {
 module.exports = {
     putNoteReorderingEntityChange,
     putEntityChangeForOtherInstances,
+    putEntityChangeWithForcedChange,
     putEntityChange,
     putEntityChangeWithInstanceId,
     fillAllEntityChanges,

src/services/erase.js

@@ -39,7 +39,7 @@ function setEntityChangesAsErased(entityChanges) {
         ec.isErased = true;
         ec.utcDateChanged = dateUtils.utcNowDateTime();
 
-        entityChangesService.putEntityChange(ec);
+        entityChangesService.putEntityChangeWithForcedChange(ec);
     }
 }
 

src/services/sync_update.js

@@ -45,10 +45,7 @@ function updateNormalEntity(remoteEC, remoteEntityRow, instanceId) {
         return false;
     }
 
-    if (!localEC
-        || localEC.utcDateChanged < remoteEC.utcDateChanged
-        || (localEC.utcDateChanged === remoteEC.utcDateChanged && localEC.hash !== remoteEC.hash) // sync error, we should still update
-    ) {
+    if (!localEC || localEC.utcDateChanged <= remoteEC.utcDateChanged) {
         if (remoteEC.entityName === 'blobs' && remoteEntityRow.content !== null) {
             // we always use a Buffer object which is different from normal saving - there we use a simple string type for
             // "string notes". The problem is that in general, it's not possible to detect whether a blob content
@@ -64,7 +61,9 @@ function updateNormalEntity(remoteEC, remoteEntityRow, instanceId) {
 
         sql.replace(remoteEC.entityName, remoteEntityRow);
 
-        entityChangesService.putEntityChangeWithInstanceId(remoteEC, instanceId);
+        if (!localEC || localEC.utcDateChanged < remoteEC.utcDateChanged) {
+            entityChangesService.putEntityChangeWithInstanceId(remoteEC, instanceId);
+        }
 
         return true;
     } else if (localEC.hash !== remoteEC.hash && localEC.utcDateChanged > remoteEC.utcDateChanged) {