From 2d2641dbd7baa267e3e803fc14c06dac5744d113 Mon Sep 17 00:00:00 2001 From: zadam Date: Sun, 9 Jan 2022 21:25:15 +0100 Subject: [PATCH] sync fixes --- .../0191__add_memberId_to_entity_changes.sql | 24 ++++++++++ db/migrations/0192__add_index_to_changeId.sql | 1 + db/schema.sql | 3 +- src/public/app/setup.js | 2 - src/routes/api/login.js | 4 +- src/routes/api/sync.js | 29 ++++++++---- src/services/app_info.js | 2 +- src/services/entity_changes.js | 12 ++--- src/services/member_id.js | 4 +- src/services/sync.js | 47 +++++++++---------- src/services/sync_update.js | 22 ++++----- src/views/setup.ejs | 4 +- 12 files changed, 90 insertions(+), 64 deletions(-) create mode 100644 db/migrations/0191__add_memberId_to_entity_changes.sql create mode 100644 db/migrations/0192__add_index_to_changeId.sql diff --git a/db/migrations/0191__add_memberId_to_entity_changes.sql b/db/migrations/0191__add_memberId_to_entity_changes.sql new file mode 100644 index 000000000..ecf132df5 --- /dev/null +++ b/db/migrations/0191__add_memberId_to_entity_changes.sql @@ -0,0 +1,24 @@ +CREATE TABLE IF NOT EXISTS "mig_entity_changes" ( + `id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + `entityName` TEXT NOT NULL, + `entityId` TEXT NOT NULL, + `hash` TEXT NOT NULL, + `isErased` INT NOT NULL, + `changeId` TEXT NOT NULL, + `componentId` TEXT NOT NULL, + `instanceId` TEXT NOT NULL, + `isSynced` INTEGER NOT NULL, + `utcDateChanged` TEXT NOT NULL +); + +INSERT INTO mig_entity_changes (id, entityName, entityId, hash, isErased, changeId, componentId, instanceId, isSynced, utcDateChanged) +SELECT id, entityName, entityId, hash, isErased, changeId, '', '', isSynced, utcDateChanged FROM entity_changes; + +DROP TABLE entity_changes; + +ALTER TABLE mig_entity_changes RENAME TO entity_changes; + +CREATE UNIQUE INDEX `IDX_entityChanges_entityName_entityId` ON "entity_changes" ( + `entityName`, + `entityId` + ); diff --git a/db/migrations/0192__add_index_to_changeId.sql b/db/migrations/0192__add_index_to_changeId.sql new file mode 100644 index 000000000..3bc24efb9 --- /dev/null +++ b/db/migrations/0192__add_index_to_changeId.sql @@ -0,0 +1 @@ +CREATE INDEX `IDX_entity_changes_changeId` ON `entity_changes` (`changeId`); diff --git a/db/schema.sql b/db/schema.sql index 76ab1955e..dd5d6efd0 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -6,7 +6,7 @@ CREATE TABLE IF NOT EXISTS "entity_changes" ( `isErased` INT NOT NULL, `changeId` TEXT NOT NULL, `componentId` TEXT NOT NULL, - `memberId` TEXT NOT NULL, + `instanceId` TEXT NOT NULL, `isSynced` INTEGER NOT NULL, `utcDateChanged` TEXT NOT NULL ); @@ -98,6 +98,7 @@ CREATE INDEX `IDX_note_revisions_utcDateCreated` ON `note_revisions` (`utcDateCr CREATE INDEX `IDX_note_revisions_utcDateLastEdited` ON `note_revisions` (`utcDateLastEdited`); CREATE INDEX `IDX_note_revisions_dateCreated` ON `note_revisions` (`dateCreated`); CREATE INDEX `IDX_note_revisions_dateLastEdited` ON `note_revisions` (`dateLastEdited`); +CREATE INDEX `IDX_entity_changes_changeId` ON `entity_changes` (`changeId`); CREATE INDEX IDX_attributes_name_value on attributes (name, value); CREATE INDEX IDX_attributes_noteId_index diff --git a/src/public/app/setup.js b/src/public/app/setup.js index 8071e502a..12d300b4f 100644 --- a/src/public/app/setup.js +++ b/src/public/app/setup.js @@ -23,8 +23,6 @@ function SetupModel() { this.syncProxy = ko.observable(); this.password = ko.observable(); - this.instanceType = utils.isElectron() ? "desktop" : "server"; - this.setupTypeSelected = () => !!this.setupType(); this.selectSetupType = () => { diff --git a/src/routes/api/login.js b/src/routes/api/login.js index 6866c3cbe..c65895f84 100644 --- a/src/routes/api/login.js +++ b/src/routes/api/login.js @@ -3,7 +3,7 @@ const options = require('../../services/options'); const utils = require('../../services/utils'); const dateUtils = require('../../services/date_utils'); -const memberId = require('../../services/member_id'); +const instanceId = require('../../services/member_id'); const passwordEncryptionService = require('../../services/password_encryption'); const protectedSessionService = require('../../services/protected_session'); const appInfo = require('../../services/app_info'); @@ -47,7 +47,7 @@ function loginSync(req) { req.session.loggedIn = true; return { - memberId: memberId, + instanceId: instanceId, maxEntityChangeId: sql.getValue("SELECT COALESCE(MAX(id), 0) FROM entity_changes WHERE isSynced = 1") }; } diff --git a/src/routes/api/sync.js b/src/routes/api/sync.js index 243a7d9f9..3276e02a5 100644 --- a/src/routes/api/sync.js +++ b/src/routes/api/sync.js @@ -124,7 +124,7 @@ function getChanged(req) { const startTime = Date.now(); let lastEntityChangeId = parseInt(req.query.lastEntityChangeId); - const clientMemberId = req.query.memberId; + const clientinstanceId = req.query.instanceId; let filteredEntityChanges = []; while (filteredEntityChanges.length === 0) { @@ -140,19 +140,28 @@ function getChanged(req) { break; } - filteredEntityChanges = entityChanges.filter(ec => ec.memberId !== clientMemberId); + filteredEntityChanges = entityChanges.filter(ec => ec.instanceId !== clientinstanceId); + + if (filteredEntityChanges.length === 0) { + lastEntityChangeId = entityChanges[entityChanges.length - 1].id; + } } - const entityChangesRecords = syncService.getEntityChangesRecords(filteredEntityChanges); + const entityChangeRecords = syncService.getEntityChangeRecords(filteredEntityChanges); - if (entityChangesRecords.length > 0) { - lastEntityChangeId = entityChangesRecords[entityChangesRecords.length - 1].entityChange.id; + if (entityChangeRecords.length > 0) { + lastEntityChangeId = entityChangeRecords[entityChangeRecords.length - 1].entityChange.id; } const ret = { - entityChanges: entityChangesRecords, - maxEntityChangeId: sql.getValue('SELECT COALESCE(MAX(id), 0) FROM entity_changes WHERE isSynced = 1'), - lastEntityChangeId + entityChanges: entityChangeRecords, + lastEntityChangeId, + outstandingPullCount: sql.getValue(` + SELECT COUNT(id) + FROM entity_changes + WHERE isSynced = 1 + AND instanceId != ? + AND id > ?`, [clientinstanceId, lastEntityChangeId]) }; if (ret.entityChanges.length > 0) { @@ -197,10 +206,10 @@ function update(req) { } } - const {entities, memberId} = body; + const {entities, instanceId} = body; for (const {entityChange, entity} of entities) { - syncUpdateService.updateEntity(entityChange, entity, memberId); + syncUpdateService.updateEntity(entityChange, entity, instanceId); } } diff --git a/src/services/app_info.js b/src/services/app_info.js index 4618c4c6c..af19d5dbb 100644 --- a/src/services/app_info.js +++ b/src/services/app_info.js @@ -4,7 +4,7 @@ const build = require('./build'); const packageJson = require('../../package'); const {TRILIUM_DATA_DIR} = require('./data_dir'); -const APP_DB_VERSION = 191; +const APP_DB_VERSION = 192; const SYNC_VERSION = 25; const CLIPPER_PROTOCOL_VERSION = "1.0"; diff --git a/src/services/entity_changes.js b/src/services/entity_changes.js index 40aa189eb..948774e51 100644 --- a/src/services/entity_changes.js +++ b/src/services/entity_changes.js @@ -3,13 +3,13 @@ const dateUtils = require('./date_utils'); const log = require('./log'); const cls = require('./cls'); const utils = require('./utils'); -const memberId = require('./member_id'); +const instanceId = require('./member_id'); const becca = require("../becca/becca"); let maxEntityChangeId = 0; -function addEntityChangeWithMemberId(origEntityChange, memberId) { - const ec = {...origEntityChange, memberId}; +function addEntityChangeWithinstanceId(origEntityChange, instanceId) { + const ec = {...origEntityChange, instanceId}; return addEntityChange(ec); } @@ -24,7 +24,7 @@ function addEntityChange(origEntityChange) { } ec.componentId = ec.componentId || cls.getComponentId() || ""; - ec.memberId = ec.memberId || memberId; + ec.instanceId = ec.instanceId || instanceId; ec.isSynced = ec.isSynced ? 1 : 0; ec.isErased = ec.isErased ? 1 : 0; ec.id = sql.replace("entity_changes", ec); @@ -43,7 +43,7 @@ function addNoteReorderingEntityChange(parentNoteId, componentId) { utcDateChanged: dateUtils.utcNowDateTime(), isSynced: true, componentId, - memberId: memberId + instanceId: instanceId }); const eventService = require('./events'); @@ -146,7 +146,7 @@ module.exports = { addNoteReorderingEntityChange, moveEntityChangeToTop, addEntityChange, - addEntityChangeWithMemberId, + addEntityChangeWithinstanceId, fillAllEntityChanges, addEntityChangesForSector, getMaxEntityChangeId: () => maxEntityChangeId diff --git a/src/services/member_id.js b/src/services/member_id.js index e08e3af38..4ee187b01 100644 --- a/src/services/member_id.js +++ b/src/services/member_id.js @@ -1,5 +1,5 @@ const utils = require('./utils'); -const memberId = utils.randomString(12); +const instanceId = utils.randomString(12); -module.exports = memberId; +module.exports = instanceId; diff --git a/src/services/sync.js b/src/services/sync.js index 4132c6c5e..dffd5d76f 100644 --- a/src/services/sync.js +++ b/src/services/sync.js @@ -4,7 +4,7 @@ const log = require('./log'); const sql = require('./sql'); const optionService = require('./options'); const utils = require('./utils'); -const memberId = require('./member_id'); +const instanceId = require('./member_id'); const dateUtils = require('./date_utils'); const syncUpdateService = require('./sync_update'); const contentHashService = require('./content_hash'); @@ -107,11 +107,11 @@ async function doLogin() { hash: hash }); - if (resp.memberId === memberId) { - throw new Error(`Sync server has member ID ${resp.memberId} which is also local. This usually happens when the sync client is (mis)configured to sync with itself (URL points back to client) instead of the correct sync server.`); + if (resp.instanceId === instanceId) { + throw new Error(`Sync server has member ID ${resp.instanceId} which is also local. This usually happens when the sync client is (mis)configured to sync with itself (URL points back to client) instead of the correct sync server.`); } - syncContext.memberId = resp.memberId; + syncContext.instanceId = resp.instanceId; const lastSyncedPull = getLastSyncedPull(); @@ -131,22 +131,21 @@ async function pullChanges(syncContext) { while (true) { const lastSyncedPull = getLastSyncedPull(); - const changesUri = `/api/sync/changed?memberId=${memberId}&lastEntityChangeId=${lastSyncedPull}`; + const changesUri = `/api/sync/changed?instanceId=${instanceId}&lastEntityChangeId=${lastSyncedPull}`; const startDate = Date.now(); const resp = await syncRequest(syncContext, 'GET', changesUri); + const {entityChanges, lastEntityChangeId} = resp; + + outstandingPullCount = resp.outstandingPullCount; const pulledDate = Date.now(); - outstandingPullCount = Math.max(0, resp.maxEntityChangeId - lastSyncedPull); - - const {entityChanges, lastEntityChangeId} = resp; - sql.transactional(() => { for (const {entityChange, entity} of entityChanges) { - const changeAppliedAlready = !entityChange.changeId - || !!sql.getValue("SELECT id FROM entity_changes WHERE changeId = ?", [entityChange.changeId]); + const changeAppliedAlready = entityChange.changeId + && !!sql.getValue("SELECT id FROM entity_changes WHERE changeId = ?", [entityChange.changeId]); if (!changeAppliedAlready) { if (!atLeastOnePullApplied) { // send only for first @@ -155,10 +154,8 @@ async function pullChanges(syncContext) { atLeastOnePullApplied = true; } - syncUpdateService.updateEntity(entityChange, entity, syncContext.memberId); + syncUpdateService.updateEntity(entityChange, entity, syncContext.instanceId); } - - outstandingPullCount = Math.max(0, resp.maxEntityChangeId - entityChange.id); } if (lastSyncedPull !== lastEntityChangeId) { @@ -191,7 +188,7 @@ async function pushChanges(syncContext) { } const filteredEntityChanges = entityChanges.filter(entityChange => { - if (entityChange.memberId === syncContext.memberId) { + if (entityChange.instanceId === syncContext.instanceId) { // this may set lastSyncedPush beyond what's actually sent (because of size limit) // so this is applied to the database only if there's no actual update lastSyncedPush = entityChange.id; @@ -211,12 +208,12 @@ async function pushChanges(syncContext) { continue; } - const entityChangesRecords = getEntityChangesRecords(filteredEntityChanges); + const entityChangesRecords = getEntityChangeRecords(filteredEntityChanges); const startDate = new Date(); await syncRequest(syncContext, 'PUT', '/api/sync/update', { entities: entityChangesRecords, - memberId + instanceId }); ws.syncPushInProgress(); @@ -253,6 +250,11 @@ async function checkContentHash(syncContext) { const failedChecks = contentHashService.checkContentHashes(resp.entityHashes); + process.exit(0); + throw new Error("AAAA"); + + return; + if (failedChecks.length > 0) { // before requeuing sectors make sure the entity changes are correct const consistencyChecks = require("./consistency_checks"); @@ -332,7 +334,7 @@ function getEntityChangeRow(entityName, entityId) { } } -function getEntityChangesRecords(entityChanges) { +function getEntityChangeRecords(entityChanges) { const records = []; let length = 0; @@ -345,13 +347,6 @@ function getEntityChangesRecords(entityChanges) { const entity = getEntityChangeRow(entityChange.entityName, entityChange.entityId); - if (entityChange.entityName === 'options' && !entity.isSynced) { - // if non-synced entities should count towards "lastSyncedPush" - records.push({entityChange}); - - continue; - } - const record = { entityChange, entity }; records.push(record); @@ -423,7 +418,7 @@ require("../becca/becca_loader").beccaLoaded.then(() => { module.exports = { sync, login, - getEntityChangesRecords, + getEntityChangeRecords, getOutstandingPullCount, getMaxEntityChangeId }; diff --git a/src/services/sync_update.js b/src/services/sync_update.js index 6a634c527..2492452e4 100644 --- a/src/services/sync_update.js +++ b/src/services/sync_update.js @@ -4,12 +4,12 @@ const entityChangesService = require('./entity_changes'); const eventService = require('./events'); const entityConstructor = require("../becca/entity_constructor"); -function updateEntity(entityChange, entityRow, memberId) { +function updateEntity(entityChange, entityRow, instanceId) { // can be undefined for options with isSynced=false if (!entityRow) { if (entityChange.isSynced) { if (entityChange.isErased) { - eraseEntity(entityChange, memberId); + eraseEntity(entityChange, instanceId); } else { log.info(`Encountered synced non-erased entity change without entity: ${JSON.stringify(entityChange)}`); @@ -23,8 +23,8 @@ function updateEntity(entityChange, entityRow, memberId) { } const updated = entityChange.entityName === 'note_reordering' - ? updateNoteReordering(entityChange, entityRow, memberId) - : updateNormalEntity(entityChange, entityRow, memberId); + ? updateNoteReordering(entityChange, entityRow, instanceId) + : updateNormalEntity(entityChange, entityRow, instanceId); if (updated) { if (entityRow.isDeleted) { @@ -42,7 +42,7 @@ function updateEntity(entityChange, entityRow, memberId) { } } -function updateNormalEntity(remoteEntityChange, entity, memberId) { +function updateNormalEntity(remoteEntityChange, entity, instanceId) { const localEntityChange = sql.getRow(` SELECT utcDateChanged, hash, isErased FROM entity_changes @@ -54,7 +54,7 @@ function updateNormalEntity(remoteEntityChange, entity, memberId) { sql.execute(`DELETE FROM ${remoteEntityChange.entityName} WHERE ${primaryKey} = ?`, remoteEntityChange.entityId); - entityChangesService.addEntityChangeWithMemberId(remoteEntityChange, memberId); + entityChangesService.addEntityChangeWithinstanceId(remoteEntityChange, instanceId); }); return true; @@ -71,7 +71,7 @@ function updateNormalEntity(remoteEntityChange, entity, memberId) { sql.transactional(() => { sql.replace(remoteEntityChange.entityName, entity); - entityChangesService.addEntityChangeWithMemberId(remoteEntityChange, memberId); + entityChangesService.addEntityChangeWithinstanceId(remoteEntityChange, instanceId); }); return true; @@ -80,13 +80,13 @@ function updateNormalEntity(remoteEntityChange, entity, memberId) { return false; } -function updateNoteReordering(entityChange, entity, memberId) { +function updateNoteReordering(entityChange, entity, instanceId) { sql.transactional(() => { for (const key in entity) { sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [entity[key], key]); } - entityChangesService.addEntityChangeWithMemberId(entityChange, memberId); + entityChangesService.addEntityChangeWithinstanceId(entityChange, instanceId); }); return true; @@ -105,7 +105,7 @@ function handleContent(content) { return content; } -function eraseEntity(entityChange, memberId) { +function eraseEntity(entityChange, instanceId) { const {entityName, entityId} = entityChange; if (!["notes", "note_contents", "branches", "attributes", "note_revisions", "note_revision_contents"].includes(entityName)) { @@ -119,7 +119,7 @@ function eraseEntity(entityChange, memberId) { eventService.emit(eventService.ENTITY_DELETE_SYNCED, { entityName, entityId }); - entityChangesService.addEntityChangeWithMemberId(entityChange, memberId); + entityChangesService.addEntityChangeWithinstanceId(entityChange, instanceId); } module.exports = { diff --git a/src/views/setup.ejs b/src/views/setup.ejs index b516bfe4f..e7f04dc19 100644 --- a/src/views/setup.ejs +++ b/src/views/setup.ejs @@ -135,9 +135,7 @@
Sync has been correctly set up. It will take some time for the initial sync to finish. Once it's done, you'll be redirected to the login page.
-
- Outstanding sync items: N/A -
+
Outstanding sync items: N/A