#98, working sync setup from server to desktop instance + refactoring of DB initialization

src/public/javascripts/setup.js

@@ -1,4 +1,3 @@
-import server from './services/server.js';
 import utils from "./services/utils.js";
 
 function SetupModel() {
@@ -56,7 +55,8 @@ function SetupModel() {
                 return;
             }
 
-            server.post('setup', {
+            // not using server.js because it loads too many dependencies
+            $.post('/api/setup/new-document', {
                 username: username,
                 password: password1
             }).then(() => {
@@ -83,7 +83,17 @@ function SetupModel() {
                 return;
             }
 
-            showAlert("All OK");
+            // not using server.js because it loads too many dependencies
+            $.post('/api/setup/sync-from-server', {
+                serverAddress: serverAddress,
+                username: username,
+                password: password
+            }).then(() => {
+                window.location.replace("/");
+            }).catch((err) => {
+                alert("Error, see dev console for details.");
+                console.error(err);
+            });
         }
     };
 }

src/routes/api/setup.js

@@ -1,13 +1,74 @@
 "use strict";
 
 const sqlInit = require('../../services/sql_init');
+const sql = require('../../services/sql');
+const cls = require('../../services/cls');
+const tmp = require('tmp-promise');
+const http = require('http');
+const fs = require('fs');
+const log = require('../../services/log');
+const DOCUMENT_PATH = require('../../services/data_dir').DOCUMENT_PATH;
+const sourceIdService = require('../../services/source_id');
+const url = require('url');
 
-async function setup(req) {
+async function setupNewDocument(req) {
     const { username, password } = req.body;
 
     await sqlInit.createInitialDatabase(username, password);
 }
 
+async function setupSyncFromServer(req) {
+    const { serverAddress, username, password } = req.body;
+
+    const tempFile = await tmp.file();
+
+    await new Promise((resolve, reject) => {
+        const file = fs.createWriteStream(tempFile.path);
+        const parsedAddress = url.parse(serverAddress);
+
+        const options = {
+            method: 'GET',
+            protocol: parsedAddress.protocol,
+            host: parsedAddress.hostname,
+            port: parsedAddress.port,
+            path: '/api/sync/document',
+            auth: username + ':' + password
+        };
+
+        log.info("Getting document from: " + serverAddress + JSON.stringify(options));
+
+        http.request(options, function(response) {
+            response.pipe(file);
+
+            file.on('finish', function() {
+                log.info("Document download finished, closing & renaming.");
+
+                file.close(() => { // close() is async, call after close completes.
+                    fs.rename(tempFile.path, DOCUMENT_PATH, async () => {
+                        cls.reset();
+
+                        await sqlInit.initDbConnection();
+
+                        // we need to generate new source ID for this instance, otherwise it will
+                        // match the original server one
+                        await sql.transactional(async () => {
+                            await sourceIdService.generateSourceId();
+                        });
+
+                        resolve();
+                    });
+                });
+            });
+        }).on('error', function(err) { // Handle errors
+            fs.unlink(tempFile.path); // Delete the file async. (But we don't check the result)
+
+            reject(err.message);
+            log.error(err.message);
+        }).end();
+    });
+}
+
 module.exports = {
-    setup
+    setupNewDocument,
+    setupSyncFromServer
 };

src/routes/api/sync.js

@@ -7,6 +7,7 @@ const sql = require('../../services/sql');
 const optionService = require('../../services/options');
 const contentHashService = require('../../services/content_hash');
 const log = require('../../services/log');
+const DOCUMENT_PATH = require('../../services/data_dir').DOCUMENT_PATH;
 
 async function checkSync() {
     return {
@@ -72,6 +73,12 @@ async function update(req) {
     }
 }
 
+async function getDocument(req, resp) {
+    log.info("Serving document.");
+
+    resp.sendFile(DOCUMENT_PATH);
+}
+
 module.exports = {
     checkSync,
     syncNow,
@@ -79,5 +86,6 @@ module.exports = {
     forceFullSync,
     forceNoteSync,
     getChanged,
-    update
+    update,
+    getDocument
 };

src/routes/routes.js

@@ -62,16 +62,21 @@ function apiRoute(method, path, routeHandler) {
     route(method, path, [auth.checkApiAuth], routeHandler, apiResultHandler);
 }
 
-function route(method, path, middleware, routeHandler, resultHandler) {
+function route(method, path, middleware, routeHandler, resultHandler, transactional = true) {
     router[method](path, ...middleware, async (req, res, next) => {
         try {
             const result = await cls.init(async () => {
                 cls.namespace.set('sourceId', req.headers.source_id);
                 protectedSessionService.setProtectedSessionId(req);
 
-                return await sql.transactional(async () => {
+                if (transactional) {
+                    return await sql.transactional(async () => {
+                        return await routeHandler(req, res, next);
+                    });
+                }
+                else {
                     return await routeHandler(req, res, next);
-                });
+                }
             });
 
             if (resultHandler) {
@@ -149,6 +154,7 @@ function register(app) {
     apiRoute(POST, '/api/sync/force-note-sync/:noteId', syncApiRoute.forceNoteSync);
     apiRoute(GET, '/api/sync/changed', syncApiRoute.getChanged);
     apiRoute(PUT, '/api/sync/update', syncApiRoute.update);
+    route(GET, '/api/sync/document', [auth.checkBasicAuth], syncApiRoute.getDocument);
 
     apiRoute(GET, '/api/event-log', eventLogRoute.getEventLog);
 
@@ -156,7 +162,8 @@ function register(app) {
     apiRoute(PUT, '/api/recent-notes/:branchId/:notePath', recentNotesRoute.addRecentNote);
     apiRoute(GET, '/api/app-info', appInfoRoute.getAppInfo);
 
-    route(POST, '/api/setup', [auth.checkAppNotInitialized], setupApiRoute.setup, apiResultHandler);
+    route(POST, '/api/setup/new-document', [auth.checkAppNotInitialized], setupApiRoute.setupNewDocument, apiResultHandler);
+    route(POST, '/api/setup/sync-from-server', [auth.checkAppNotInitialized], setupApiRoute.setupSyncFromServer, apiResultHandler, false);
 
     apiRoute(POST, '/api/sql/execute', sqlRoute.execute);
     apiRoute(POST, '/api/anonymization/anonymize', anonymizationRoute.anonymize);

src/services/auth.js

@@ -1,12 +1,13 @@
 "use strict";
 
-const migrationService = require('./migration');
 const sql = require('./sql');
 const sqlInit = require('./sql_init');
 const utils = require('./utils');
+const passwordEncryptionService = require('./password_encryption');
+const optionService = require('./options');
 
 async function checkAuth(req, res, next) {
-    if (!await sqlInit.isUserInitialized()) {
+    if (!await sqlInit.isDbInitialized()) {
         res.redirect("setup");
     }
     else if (!req.session.loggedIn && !utils.isElectron()) {
@@ -38,7 +39,7 @@ async function checkApiAuth(req, res, next) {
 }
 
 async function checkAppNotInitialized(req, res, next) {
-    if (await sqlInit.isUserInitialized()) {
+    if (await sqlInit.isDbInitialized()) {
         res.status(400).send("App already initialized.");
     }
     else {
@@ -57,10 +58,27 @@ async function checkSenderToken(req, res, next) {
     }
 }
 
+async function checkBasicAuth(req, res, next) {
+    const header = req.headers.authorization || '';
+    const token = header.split(/\s+/).pop() || '';
+    const auth = new Buffer.from(token, 'base64').toString();
+    const [username, password] = auth.split(/:/);
+
+    const dbUsername = await optionService.getOption('username');
+
+    if (dbUsername !== username || !await passwordEncryptionService.verifyPassword(password)) {
+        res.status(401).send("Not authorized");
+    }
+    else {
+        next();
+    }
+}
+
 module.exports = {
     checkAuth,
     checkApiAuth,
     checkAppNotInitialized,
     checkApiAuthOrElectron,
-    checkSenderToken
+    checkSenderToken,
+    checkBasicAuth
 };

src/services/cls.js

@@ -13,9 +13,14 @@ function getSourceId() {
     return namespace.get('sourceId');
 }
 
+function reset() {
+    clsHooked.reset();
+}
+
 module.exports = {
     init,
     wrap,
     namespace,
-    getSourceId
+    getSourceId,
+    reset
 };

src/services/migration.js

@@ -86,7 +86,7 @@ async function migrate() {
     }
 
     if (sqlInit.isDbUpToDate()) {
-        sqlInit.setDbReadyAsResolved();
+        await sqlInit.initDbConnection();
     }
 
     return migrations;

src/services/sql.js

@@ -157,10 +157,10 @@ async function transactional(func) {
     transactionActive = true;
     transactionPromise = new Promise(async (resolve, reject) => {
         try {
-            cls.namespace.set('isInTransaction', true);
-
             await beginTransaction();
 
+            cls.namespace.set('isInTransaction', true);
+
             ret = await func();
 
             await commit();

src/services/sql_init.js

@@ -11,38 +11,32 @@ async function createConnection() {
     return await sqlite.open(dataDir.DOCUMENT_PATH, {Promise});
 }
 
-let schemaReadyResolve = null;
-const schemaReady = new Promise((resolve, reject) => schemaReadyResolve = resolve);
-
 let dbReadyResolve = null;
 const dbReady = new Promise((resolve, reject) => {
-    cls.init(async () => {
+    dbReadyResolve = resolve;
+
+    initDbConnection();
+});
+
+async function isDbInitialized() {
+    const tableResults = await sql.getRows("SELECT name FROM sqlite_master WHERE type='table' AND name='notes'");
+
+    return tableResults.length === 1;
+}
+
+async function initDbConnection() {
+    await cls.init(async () => {
         const db = await createConnection();
         sql.setDbConnection(db);
 
         await sql.execute("PRAGMA foreign_keys = ON");
 
-        dbReadyResolve = () => {
-            log.info("DB ready.");
-
-            resolve(db);
-        };
-
-        const tableResults = await sql.getRows("SELECT name FROM sqlite_master WHERE type='table' AND name='notes'");
-        if (tableResults.length !== 1) {
+        if (isDbInitialized()) {
             log.info("DB not found, please visit setup page to initialize Trilium.");
 
             return;
         }
 
-        schemaReadyResolve();
-
-        if (!await isUserInitialized()) {
-            log.info("Login/password not initialized. DB not ready.");
-
-            return;
-        }
-
         if (!await isDbUpToDate()) {
             // avoiding circular dependency
             const migrationService = require('./migration');
@@ -50,9 +44,10 @@ const dbReady = new Promise((resolve, reject) => {
             await migrationService.migrate();
         }
 
-        resolve(db);
+        log.info("DB ready.");
+        dbReadyResolve(db);
     });
-});
+}
 
 async function createInitialDatabase(username, password) {
     log.info("Connected to db, but schema doesn't exist. Initializing schema ...");
@@ -78,11 +73,7 @@ async function createInitialDatabase(username, password) {
 
     log.info("Schema and initial content generated. Waiting for user to enter username/password to finish setup.");
 
-    setDbReadyAsResolved();
-}
-
-function setDbReadyAsResolved() {
-    dbReadyResolve();
+    await initDbConnection();
 }
 
 async function isDbUpToDate() {
@@ -97,23 +88,10 @@ async function isDbUpToDate() {
     return upToDate;
 }
 
-async function isUserInitialized() {
-    const optionsTable = await sql.getRows("SELECT name FROM sqlite_master WHERE type='table' AND name='options'");
-
-    if (optionsTable.length !== 1) {
-        return false;
-    }
-
-    const username = await sql.getValue("SELECT value FROM options WHERE name = 'username'");
-
-    return !!username;
-}
-
 module.exports = {
     dbReady,
-    schemaReady,
-    isUserInitialized,
-    setDbReadyAsResolved,
+    isDbInitialized,
+    initDbConnection,
     isDbUpToDate,
     createInitialDatabase
 };

src/tools/generate_document.js

@@ -28,7 +28,7 @@ async function setUserNamePassword() {
 
     await passwordEncryptionService.setDataKey(password, utils.randomSecureToken(16));
 
-    sqlInit.setDbReadyAsResolved();
+    await sqlInit.initDbConnection();
 }
 
 const noteCount = parseInt(process.argv[2]);
@@ -71,4 +71,4 @@ async function start() {
     process.exit(0);
 }
 
-sqlInit.schemaReady.then(cls.wrap(start));
+sqlInit.dbReady.then(cls.wrap(start));