From 0ec909fd7a908cd398f45f4c29e99d582f9dd875 Mon Sep 17 00:00:00 2001 From: azivner Date: Wed, 28 Mar 2018 23:41:22 -0400 Subject: [PATCH] added basic CLS support with re-entrant transactions --- package-lock.json | 36 +++++++++++++++++++ package.json | 1 + src/app.js | 12 +++++++ .../javascripts/services/note_detail.js | 6 +++- src/routes/api/notes.js | 4 ++- src/services/backup.js | 5 +-- src/services/cls.js | 16 +++++++++ src/services/consistency_checks.js | 5 +-- src/services/notes.js | 20 +++++------ src/services/scheduler.js | 7 ++-- src/services/source_id.js | 3 +- src/services/sql.js | 19 ++++++++-- src/services/sync.js | 5 +-- 13 files changed, 115 insertions(+), 24 deletions(-) create mode 100644 src/services/cls.js diff --git a/package-lock.json b/package-lock.json index 743bd4d7a..ee707d0c0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -395,6 +395,14 @@ "resolved": "https://registry.npmjs.org/async-each-series/-/async-each-series-1.1.0.tgz", "integrity": "sha1-9C/YFV048hpbjqB8KOBj7RcAsTg=" }, + "async-hook-jl": { + "version": "1.7.6", + "resolved": "https://registry.npmjs.org/async-hook-jl/-/async-hook-jl-1.7.6.tgz", + "integrity": "sha512-gFaHkFfSxTjvoxDMYqDuGHlcRyUuamF8s+ZTtJdDzqjws4mCt7v0vuV79/E2Wr2/riMQgtG4/yUtXWs1gZ7JMg==", + "requires": { + "stack-chain": "1.3.7" + } + }, "async-limiter": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", @@ -1888,6 +1896,16 @@ "resolved": "https://registry.npmjs.org/clone-stats/-/clone-stats-0.0.1.tgz", "integrity": "sha1-uI+UqCzzi4eR1YBG6kAprYjKmdE=" }, + "cls-hooked": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/cls-hooked/-/cls-hooked-4.2.2.tgz", + "integrity": "sha512-J4Xj5f5wq/4jAvcdgoGsL3G103BtWpZrMo8NEinRltN+xpTZdI+M38pyQqhuFU/P792xkMFvnKSf+Lm81U1bxw==", + "requires": { + "async-hook-jl": "1.7.6", + "emitter-listener": "1.1.1", + "semver": "5.4.1" + } + }, "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -3593,6 +3611,14 @@ "integrity": "sha1-H71tl779crim+SHcONIkE9L2/d8=", "dev": true }, + "emitter-listener": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.1.tgz", + "integrity": "sha1-6Lu+gkS8jg0LTvcc0UKUx/JBx+w=", + "requires": { + "shimmer": "1.2.0" + } + }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -9677,6 +9703,11 @@ "rechoir": "0.6.2" } }, + "shimmer": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.0.tgz", + "integrity": "sha512-xTCx2vohXC2EWWDqY/zb4+5Mu28D+HYNSOuFzsyRDRvI/e1ICb69afwaUwfjr+25ZXldbOLyp+iDUZHq8UnTag==" + }, "signal-exit": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", @@ -10580,6 +10611,11 @@ "tweetnacl": "0.14.5" } }, + "stack-chain": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/stack-chain/-/stack-chain-1.3.7.tgz", + "integrity": "sha1-0ZLJ/06moiyUxN1FkXHj8AzqEoU=" + }, "stat-mode": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/stat-mode/-/stat-mode-0.2.2.tgz", diff --git a/package.json b/package.json index 044f78229..b905bc52f 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "async-mutex": "^0.1.3", "axios": "^0.17.1", "body-parser": "~1.18.2", + "cls-hooked": "^4.2.2", "cookie-parser": "~1.4.3", "debug": "~3.1.0", "devtron": "^1.4.0", diff --git a/src/app.js b/src/app.js index 38936ff5e..23ecb08bc 100644 --- a/src/app.js +++ b/src/app.js @@ -9,6 +9,7 @@ const session = require('express-session'); const FileStore = require('session-file-store')(session); const os = require('os'); const sessionSecret = require('./services/session_secret'); +const cls = require('./services/cls'); const app = express(); @@ -23,6 +24,17 @@ app.use((req, res, next) => { next(); }); +app.use((req, res, next) => { + cls.namespace.bindEmitter(req); + cls.namespace.bindEmitter(res); + + cls.init(() => { + cls.namespace.set("Hi"); + + next(); + }); +}); + app.use(bodyParser.json({limit: '50mb'})); app.use(bodyParser.urlencoded({extended: false})); app.use(cookieParser()); diff --git a/src/public/javascripts/services/note_detail.js b/src/public/javascripts/services/note_detail.js index 76141bd71..33d45cf4b 100644 --- a/src/public/javascripts/services/note_detail.js +++ b/src/public/javascripts/services/note_detail.js @@ -108,7 +108,11 @@ function updateNoteFromInputs(note) { } async function saveNoteToServer(note) { - await server.put('notes/' + note.noteId, note); + const dto = Object.assign({}, note); + delete dto.treeCache; + delete dto.hideInAutocomplete; + + await server.put('notes/' + dto.noteId, dto); isNoteChanged = false; diff --git a/src/routes/api/notes.js b/src/routes/api/notes.js index 9b9bebc43..48561254a 100644 --- a/src/routes/api/notes.js +++ b/src/routes/api/notes.js @@ -55,7 +55,9 @@ router.put('/:noteId', auth.checkApiAuth, wrap(async (req, res, next) => { const sourceId = req.headers.source_id; const dataKey = protected_session.getDataKey(req); - await notes.updateNote(noteId, note, dataKey, sourceId); + await sql.doInTransaction(async () => { + await notes.updateNote(noteId, note, dataKey, sourceId); + }); res.send({}); })); diff --git a/src/services/backup.js b/src/services/backup.js index 085aa1075..029723712 100644 --- a/src/services/backup.js +++ b/src/services/backup.js @@ -7,6 +7,7 @@ const dataDir = require('./data_dir'); const log = require('./log'); const sql = require('./sql'); const sync_mutex = require('./sync_mutex'); +const cls = require('./cls'); async function regularBackup() { const now = new Date(); @@ -64,10 +65,10 @@ if (!fs.existsSync(dataDir.BACKUP_DIR)) { } sql.dbReady.then(() => { - setInterval(regularBackup, 60 * 60 * 1000); + setInterval(cls.wrap(regularBackup), 60 * 60 * 1000); // kickoff backup immediately - setTimeout(regularBackup, 1000); + setTimeout(cls.wrap(regularBackup), 1000); }); module.exports = { diff --git a/src/services/cls.js b/src/services/cls.js new file mode 100644 index 000000000..5bcb7b2bb --- /dev/null +++ b/src/services/cls.js @@ -0,0 +1,16 @@ +const clsHooked = require('cls-hooked'); +const namespace = clsHooked.createNamespace("trilium"); + +async function init(callback) { + return await namespace.runAndReturn(callback); +} + +function wrap(callback) { + return async () => await init(callback); +} + +module.exports = { + init, + wrap, + namespace +}; \ No newline at end of file diff --git a/src/services/consistency_checks.js b/src/services/consistency_checks.js index 566e4660f..666a20a8d 100644 --- a/src/services/consistency_checks.js +++ b/src/services/consistency_checks.js @@ -5,6 +5,7 @@ const log = require('./log'); const messaging = require('./messaging'); const sync_mutex = require('./sync_mutex'); const utils = require('./utils'); +const cls = require('./cls'); async function runCheck(query, errorText, errorList) { utils.assertArguments(query, errorText, errorList); @@ -268,10 +269,10 @@ async function runChecks() { } sql.dbReady.then(() => { - setInterval(runChecks, 60 * 60 * 1000); + setInterval(cls.wrap(runChecks), 60 * 60 * 1000); // kickoff backup immediately - setTimeout(runChecks, 10000); + setTimeout(cls.wrap(runChecks), 10000); }); module.exports = { diff --git a/src/services/notes.js b/src/services/notes.js index c6814352f..b277207ca 100644 --- a/src/services/notes.js +++ b/src/services/notes.js @@ -262,16 +262,16 @@ async function loadFile(noteId, newNote, dataKey) { await protected_session.decryptNote(dataKey, oldNote); } - newNote.detail.content = oldNote.content; + newNote.content = oldNote.content; } async function updateNote(noteId, newNote, dataKey, sourceId) { - if (newNote.detail.type === 'file') { + if (newNote.type === 'file') { await loadFile(noteId, newNote, dataKey); } - if (newNote.detail.isProtected) { - await protected_session.encryptNote(dataKey, newNote.detail); + if (newNote.isProtected) { + await protected_session.encryptNote(dataKey, newNote); } const labelsMap = await labels.getNoteLabelMap(noteId); @@ -287,7 +287,7 @@ async function updateNote(noteId, newNote, dataKey, sourceId) { "SELECT noteRevisionId FROM note_revisions WHERE noteId = ? AND dateModifiedTo >= ?", [noteId, revisionCutoff]); await sql.doInTransaction(async () => { - const msSinceDateCreated = now.getTime() - utils.parseDateTime(newNote.detail.dateCreated).getTime(); + const msSinceDateCreated = now.getTime() - utils.parseDateTime(newNote.dateCreated).getTime(); if (labelsMap.disable_versioning !== 'true' && !existingnoteRevisionId @@ -296,14 +296,14 @@ async function updateNote(noteId, newNote, dataKey, sourceId) { await saveNoteRevision(noteId, dataKey, sourceId, nowStr); } - await saveNoteImages(noteId, newNote.detail.content, sourceId); + await saveNoteImages(noteId, newNote.content, sourceId); - await protectNoteRevisions(noteId, dataKey, newNote.detail.isProtected); + await protectNoteRevisions(noteId, dataKey, newNote.isProtected); await sql.execute("UPDATE notes SET title = ?, content = ?, isProtected = ?, dateModified = ? WHERE noteId = ?", [ - newNote.detail.title, - newNote.detail.content, - newNote.detail.isProtected, + newNote.title, + newNote.content, + newNote.isProtected, nowStr, noteId]); diff --git a/src/services/scheduler.js b/src/services/scheduler.js index 26e0083ef..102ed120d 100644 --- a/src/services/scheduler.js +++ b/src/services/scheduler.js @@ -1,5 +1,6 @@ const script = require('./script'); const Repository = require('./repository'); +const cls = require('./cls'); const repo = new Repository(); @@ -20,8 +21,8 @@ async function runNotesWithLabel(runAttrValue) { } } -setTimeout(() => runNotesWithLabel('backend_startup'), 10 * 1000); +setTimeout(cls.wrap(() => runNotesWithLabel('backend_startup')), 10 * 1000); -setInterval(() => runNotesWithLabel('hourly'), 3600 * 1000); +setInterval(cls.wrap(() => runNotesWithLabel('hourly')), 3600 * 1000); -setInterval(() => runNotesWithLabel('daily'), 24 * 3600 * 1000); \ No newline at end of file +setInterval(cls.wrap(() => runNotesWithLabel('daily'), 24 * 3600 * 1000)); \ No newline at end of file diff --git a/src/services/source_id.js b/src/services/source_id.js index cd425d3ec..f5b89f4f1 100644 --- a/src/services/source_id.js +++ b/src/services/source_id.js @@ -1,6 +1,7 @@ const utils = require('./utils'); const log = require('./log'); const sql = require('./sql'); +const cls = require('./cls'); async function saveSourceId(sourceId) { await sql.doInTransaction(async () => { @@ -41,7 +42,7 @@ function isLocalSourceId(srcId) { const currentSourceId = createSourceId(); // this will also refresh source IDs -sql.dbReady.then(() => saveSourceId(currentSourceId)); +sql.dbReady.then(cls.wrap(() => saveSourceId(currentSourceId))); function getCurrentSourceId() { return currentSourceId; diff --git a/src/services/sql.js b/src/services/sql.js index 6cb1d65b4..c30e6f53a 100644 --- a/src/services/sql.js +++ b/src/services/sql.js @@ -6,6 +6,7 @@ const fs = require('fs'); const sqlite = require('sqlite'); const app_info = require('./app_info'); const resource_dir = require('./resource_dir'); +const cls = require('./cls'); async function createConnection() { return await sqlite.open(dataDir.DOCUMENT_PATH, {Promise}); @@ -15,7 +16,7 @@ const dbConnected = createConnection(); let dbReadyResolve = null; const dbReady = new Promise((resolve, reject) => { - dbConnected.then(async db => { + dbConnected.then(cls.wrap(async db => { await execute("PRAGMA foreign_keys = ON"); dbReadyResolve = () => { @@ -65,7 +66,7 @@ const dbReady = new Promise((resolve, reject) => { resolve(db); } - }) + })) .catch(e => { console.log("Error connecting to DB.", e); process.exit(1); @@ -191,6 +192,15 @@ let transactionActive = false; let transactionPromise = null; async function doInTransaction(func) { + if (cls.namespace.get('isInTransaction')) { + console.log("Transaction already active"); + + return await func(); + } + else { + console.log("Starting new transaction"); + } + while (transactionActive) { await transactionPromise; } @@ -201,6 +211,8 @@ async function doInTransaction(func) { transactionActive = true; transactionPromise = new Promise(async (resolve, reject) => { try { + cls.namespace.set('isInTransaction', true); + await beginTransaction(); ret = await func(); @@ -219,6 +231,9 @@ async function doInTransaction(func) { reject(e); } + finally { + cls.namespace.set('isInTransaction', false); + } }); if (transactionActive) { diff --git a/src/services/sync.js b/src/services/sync.js index c22387c7f..8c7fc9d1d 100644 --- a/src/services/sync.js +++ b/src/services/sync.js @@ -15,6 +15,7 @@ const app_info = require('./app_info'); const messaging = require('./messaging'); const sync_setup = require('./sync_setup'); const sync_mutex = require('./sync_mutex'); +const cls = require('./cls'); let proxyToggle = true; let syncServerCertificate = null; @@ -347,10 +348,10 @@ sql.dbReady.then(() => { syncServerCertificate = fs.readFileSync(sync_setup.SYNC_CERT_PATH); } - setInterval(sync, 60000); + setInterval(cls.wrap(sync), 60000); // kickoff initial sync immediately - setTimeout(sync, 1000); + setTimeout(cls.wrap(sync), 1000); } else { log.info("Sync server not configured, sync timer not running.")