sascha
/
Notes
mirror of https://github.com/TriliumNext/Notes
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

implemented bulk sync pull for increased performance

pull/255/head
azivner 2018-04-07 21:53:42 +07:00
parent b09463d1b2
commit 64336ffbee
2 changed files with 62 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
src/routes/api/sync.js
Unescape Escape View File

@ -58,7 +58,52 @@ async function forceNoteSync(req) {
async function getChanged(req) {
const lastSyncId = parseInt(req.query.lastSyncId);
return await sql.getRows("SELECT * FROM sync WHERE id > ?", [lastSyncId]);
const records = [];
let length = 0;
for (const sync of await sql.getRows("SELECT * FROM sync WHERE id > ?", [lastSyncId])) {
const record = {
sync: sync,
entity: await getEntityRow(sync.entityName, sync.entityId)
};
records.push(record);
length += JSON.stringify(record).length;
if (length > 1000000) {
break;
}
}
return records;
}
const primaryKeys = {
"notes": "noteId",
"branches": "branchId",
"note_revisions": "noteRevisionId",
"option": "name",
"recent_notes": "branchId",
"images": "imageId",
"note_images": "noteImageId",
"labels": "labelId",
"api_tokens": "apiTokenId"
};
async function getEntityRow(entityName, entityId) {
if (entityName === 'note_reordering') {
return await getNoteReordering(entityId);
}
else {
const primaryKey = primaryKeys[entityName];
if (!primaryKey) {
throw new Error("Unknown entity " + entityName);
}
return await sql.getRow(`SELECT * FROM ${entityName} WHERE ${primaryKey} = ?`, [entityId]);
}
}
async function getNote(req) {
@ -96,13 +141,8 @@ async function getOption(req) {
}
}
async function getNoteReordering(req) {
const parentNoteId = req.params.parentNoteId;
return {
parentNoteId: parentNoteId,
ordering: await sql.getMap("SELECT branchId, notePosition FROM branches WHERE parentNoteId = ? AND isDeleted = 0", [parentNoteId])
};
async function getNoteReordering(parentNoteId) {
return await sql.getMap("SELECT branchId, notePosition FROM branches WHERE parentNoteId = ? AND isDeleted = 0", [parentNoteId])
}
async function getRecentNote(req) {

30
src/services/sync.js
Unescape Escape View File

@ -104,11 +104,11 @@ async function pullSync(syncContext) {
const changesUri = '/api/sync/changed?lastSyncId=' + lastSyncedPull;
const syncRows = await syncRequest(syncContext, 'GET', changesUri);
const rows = await syncRequest(syncContext, 'GET', changesUri);
log.info("Pulled " + syncRows.length + " changes from " + changesUri);
log.info("Pulled " + rows.length + " changes from " + changesUri);
for (const sync of syncRows) {
for (const {sync, entity} of rows) {
if (sourceIdService.isLocalSourceId(sync.sourceId)) {
log.info(`Skipping pull #${sync.id} ${sync.entityName} ${sync.entityId} because ${sync.sourceId} is a local source id.`);
@ -117,40 +117,38 @@ async function pullSync(syncContext) {
continue;
}
const resp = await syncRequest(syncContext, 'GET', "/api/sync/" + sync.entityName + "/" + encodeURIComponent(sync.entityId));
if (!resp || (sync.entityName === 'notes' && !resp.entity)) {
if (!entity) {
log.error(`Empty response to pull for sync #${sync.id} ${sync.entityName}, id=${sync.entityId}`);
}
else if (sync.entityName === 'notes') {
await syncUpdateService.updateNote(resp.entity, syncContext.sourceId);
await syncUpdateService.updateNote(entity, syncContext.sourceId);
}
else if (sync.entityName === 'branches') {
await syncUpdateService.updateBranch(resp, syncContext.sourceId);
await syncUpdateService.updateBranch(entity, syncContext.sourceId);
}
else if (sync.entityName === 'note_revisions') {
await syncUpdateService.updateNoteRevision(resp, syncContext.sourceId);
await syncUpdateService.updateNoteRevision(entity, syncContext.sourceId);
}
else if (sync.entityName === 'note_reordering') {
await syncUpdateService.updateNoteReordering(resp, syncContext.sourceId);
await syncUpdateService.updateNoteReordering(entity, syncContext.sourceId);
}
else if (sync.entityName === 'options') {
await syncUpdateService.updateOptions(resp, syncContext.sourceId);
await syncUpdateService.updateOptions(entity, syncContext.sourceId);
}
else if (sync.entityName === 'recent_notes') {
await syncUpdateService.updateRecentNotes(resp, syncContext.sourceId);
await syncUpdateService.updateRecentNotes(entity, syncContext.sourceId);
}
else if (sync.entityName === 'images') {
await syncUpdateService.updateImage(resp, syncContext.sourceId);
await syncUpdateService.updateImage(entity, syncContext.sourceId);
}
else if (sync.entityName === 'note_images') {
await syncUpdateService.updateNoteImage(resp, syncContext.sourceId);
await syncUpdateService.updateNoteImage(entity, syncContext.sourceId);
}
else if (sync.entityName === 'labels') {
await syncUpdateService.updateLabel(resp, syncContext.sourceId);
await syncUpdateService.updateLabel(entity, syncContext.sourceId);
}
else if (sync.entityName === 'api_tokens') {
await syncUpdateService.updateApiToken(resp, syncContext.sourceId);
await syncUpdateService.updateApiToken(entity, syncContext.sourceId);
}
else {
throw new Error(`Unrecognized entity type ${sync.entityName} in sync #${sync.id}`);