sync WIP

bin/www

@@ -1,26 +1,28 @@
 #!/usr/bin/env node
 
 process.on('unhandledRejection', (reason, p) => {
+    const message = 'Unhandled Rejection at: Promise' + p + ', reason:' + reason;
     // this makes sure that stacktrace of failed promise is printed out
-    console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
+    console.log(message);
+
+    // but also try to log it into file
+    require('../services/log').error(message);
 });
 
-var app = require('../app');
-var debug = require('debug')('node:server');
-var http = require('http');
+const app = require('../app');
+const debug = require('debug')('node:server');
+const http = require('http');
 
 /**
  * Get port from environment and store in Express.
  */
-
-var port = normalizePort(process.env.PORT || '3000');
+const port = normalizePort(process.env.PORT || '3000');
 app.set('port', port);
 
 /**
  * Create HTTP server.
  */
-
-var server = http.createServer(app);
+const server = http.createServer(app);
 
 /**
  * Listen on provided port, on all network interfaces.
@@ -35,19 +37,19 @@ server.on('listening', onListening);
  */
 
 function normalizePort(val) {
-  var port = parseInt(val, 10);
+    const port = parseInt(val, 10);
 
-  if (isNaN(port)) {
-    // named pipe
-    return val;
-  }
+    if (isNaN(port)) {
+        // named pipe
+        return val;
+    }
 
-  if (port >= 0) {
-    // port number
-    return port;
-  }
+    if (port >= 0) {
+        // port number
+        return port;
+    }
 
-  return false;
+    return false;
 }
 
 /**
@@ -55,27 +57,29 @@ function normalizePort(val) {
  */
 
 function onError(error) {
-  if (error.syscall !== 'listen') {
-    throw error;
-  }
+    if (error.syscall !== 'listen') {
+        throw error;
+    }
 
-  var bind = typeof port === 'string'
-    ? 'Pipe ' + port
-    : 'Port ' + port;
+    const bind = typeof port === 'string'
+        ? 'Pipe ' + port
+        : 'Port ' + port;
 
-  // handle specific listen errors with friendly messages
-  switch (error.code) {
-    case 'EACCES':
-      console.error(bind + ' requires elevated privileges');
-      process.exit(1);
-      break;
-    case 'EADDRINUSE':
-      console.error(bind + ' is already in use');
-      process.exit(1);
-      break;
-    default:
-      throw error;
-  }
+    // handle specific listen errors with friendly messages
+    switch (error.code) {
+        case 'EACCES':
+            console.error(bind + ' requires elevated privileges');
+            process.exit(1);
+        break;
+
+        case 'EADDRINUSE':
+            console.error(bind + ' is already in use');
+            process.exit(1);
+        break;
+
+        default:
+            throw error;
+    }
 }
 
 /**
@@ -83,9 +87,10 @@ function onError(error) {
  */
 
 function onListening() {
-  var addr = server.address();
-  var bind = typeof addr === 'string'
-    ? 'pipe ' + addr
-    : 'port ' + addr.port;
-  debug('Listening on ' + bind);
+    const addr = server.address();
+    const bind = typeof addr === 'string'
+        ? 'pipe ' + addr
+        : 'port ' + addr.port;
+
+    debug('Listening on ' + bind);
 }

migrations/0011__add_last_synced_option.sql

@@ -0,0 +1 @@
+INSERT INTO options (opt_name, opt_value) VALUES ('last_synced', 0)

package.json

@@ -22,6 +22,8 @@
     "fs-extra": "^4.0.2",
     "helmet": "^3.9.0",
     "ini": "^1.3.4",
+    "request": "^2.83.0",
+    "request-promise": "^4.2.2",
     "scrypt": "^6.0.3",
     "serve-favicon": "~2.4.5",
     "session-file-store": "^1.1.2",

routes/api/sync.js

@@ -10,7 +10,21 @@ router.get('/changed/:since', auth.checkApiAuth, async (req, res, next) => {
 
     res.send({
         'tree': await sql.getResults("select * from notes_tree where date_modified >= ?", [since]),
-        'notes': await sql.getFlattenedResults('note_id', "select note_id from notes where date_modified >= ?", [since])
+        'notes': await sql.getFlattenedResults('note_id', "select note_id from notes where date_modified >= ?", [since]),
+        'audit_log': await sql.getResults("select * from audit_log where date_modified >= ?", [since])
+    });
+});
+
+router.get('/note/:noteId/:since', auth.checkApiAuth, async (req, res, next) => {
+    const noteId = req.params.noteId;
+    const since = parseInt(req.params.since);
+
+    const detail = await sql.getSingleResult("select * from notes where note_id = ?", [noteId]);
+
+    res.send({
+        'detail': detail,
+        'images': await sql.getResults("select * from images where note_id = ? order by note_offset", [noteId]),
+        'history': await sql.getResults("select * from notes_history where note_id = ? and date_modified_to >= ?", [noteId, since])
     });
 });
 

routes/index.js

@@ -7,14 +7,7 @@ const migration = require('../services/migration');
 const sql = require('../services/sql');
 
 router.get('', auth.checkAuth, async (req, res, next) => {
-    const dbVersion = parseInt(await sql.getOption('db_version'))
-
-    if (dbVersion < migration.APP_DB_VERSION) {
-        res.redirect("migration");
-    }
-    else {
-        res.render('index', {});
-    }
+    res.render('index', {});
 });
 
 module.exports = router;

services/auth.js

@@ -1,19 +1,31 @@
 "use strict";
 
-function checkAuth(req, res, next) {
+const migration = require('./migration');
+
+async function checkAuth(req, res, next) {
     if (!req.session.loggedIn) {
         res.redirect("login");
-    } else {
+    }
+
+    if (await migration.isDbUpToDate()) {
         next();
     }
+    else {
+        res.redirect("migration");
+    }
 }
 
-function checkApiAuth(req, res, next) {
-    if (!req.session.loggedIn) {
+async function checkApiAuth(req, res, next) {
+    if (!req.session.loggedIn && req.header("auth") !== "sync") {
         res.sendStatus(401);
-    } else {
+    }
+
+    if (await migration.isDbUpToDate()) {
         next();
     }
+    else {
+        res.sendStatus(409); // need better response than that
+    }
 }
 
 module.exports = {

services/migration.js

@@ -3,7 +3,7 @@ const sql = require('./sql');
 const fs = require('fs-extra');
 const log = require('./log');
 
-const APP_DB_VERSION = 10;
+const APP_DB_VERSION = 11;
 const MIGRATIONS_DIR = "./migrations";
 
 async function migrate() {
@@ -67,7 +67,14 @@ async function migrate() {
     return migrations;
 }
 
+async function isDbUpToDate() {
+    const dbVersion = parseInt(await sql.getOption('db_version'));
+
+    return dbVersion >= APP_DB_VERSION;
+}
+
 module.exports = {
     migrate,
+    isDbUpToDate,
     APP_DB_VERSION
 };

services/sql.js

@@ -4,11 +4,19 @@ const db = require('sqlite');
 const utils = require('./utils');
 const log = require('./log');
 
-async function insert(table_name, rec) {
-    const columns = Object.keys(rec).join(", ");
-    const questionMarks = Object.keys(rec).map(p => "?").join(", ");
+async function insert(table_name, rec, replace = false) {
+    const keys = Object.keys(rec);
+    if (keys.length === 0) {
+        log.error("Can't insert empty object into table " + table_name);
+        return;
+    }
 
-    const res = await execute("INSERT INTO " + table_name + "(" + columns + ") VALUES (" + questionMarks + ")", Object.values(rec));
+    const columns = keys.join(", ");
+    const questionMarks = keys.map(p => "?").join(", ");
+
+    const query = "INSERT " + (replace ? "OR REPLACE" : "") + " INTO " + table_name + "(" + columns + ") VALUES (" + questionMarks + ")";
+
+    const res = await execute(query, Object.values(rec));
 
     return res.lastID;
 }
@@ -21,6 +29,10 @@ async function commit() {
     return await db.run("COMMIT");
 }
 
+async function rollback() {
+    return await db.run("ROLLBACK");
+}
+
 async function getOption(optName) {
     const row = await getSingleResult("SELECT opt_value FROM options WHERE opt_name = ?", [optName]);
 
@@ -94,6 +106,7 @@ module.exports = {
     setOption,
     beginTransaction,
     commit,
+    rollback,
     addAudit,
     deleteRecentAudits,
     remove

services/sync.js

@@ -1,7 +1,94 @@
 "use strict";
 
-function sync() {
+const log = require('./log');
+const rp = require('request-promise');
+const sql = require('./sql');
+const migration = require('./migration');
 
+const SYNC_SERVER = 'http://localhost:3000';
+
+
+let syncInProgress = false;
+
+async function sync() {
+    try {
+        syncInProgress = true;
+
+        if (!await migration.isDbUpToDate()) {
+            return;
+        }
+
+        const lastSynced = parseInt(await sql.getOption('last_synced'));
+
+        const resp = await rp({
+            uri: SYNC_SERVER + '/api/sync/changed/' + lastSynced,
+            headers: {
+                auth: 'sync'
+            },
+            json: true
+        });
+
+        try {
+            sql.beginTransaction();
+
+            for (const treeItem of resp.tree) {
+                delete treeItem['id'];
+
+                await sql.insert("notes_tree", treeItem, true);
+
+                log.info("Syncing notes_tree " + treeItem.note_id);
+            }
+
+            for (const audit of resp.audit_log) {
+                delete audit['id'];
+
+                await sql.insert("audit_log", audit, true);
+
+                log.info("Syncing audit_log for noteId=" + audit.note_id);
+            }
+
+            for (const noteId of resp.notes) {
+                const note = await rp({
+                    uri: SYNC_SERVER + "/api/sync/note/" + noteId + "/" + lastSynced,
+                    headers: {
+                        auth: 'sync'
+                    },
+                    json: true
+                });
+
+                console.log(noteId);
+
+                await sql.insert("notes", note.detail, true);
+
+                await sql.remove("images", noteId);
+
+                for (const image of note.images) {
+                    await sql.insert("images", image);
+                }
+
+                for (const history of note.history) {
+                    delete history['id'];
+
+                    await sql.insert("notes_history", history);
+                }
+            }
+
+            sql.commit();
+        }
+        catch (e) {
+            sql.rollback();
+
+            throw e;
+        }
+    }
+    catch (e) {
+        log.error("sync failed: " + e.stack);
+    }
+    finally {
+        syncInProgress = false;
+    }
 }
 
-setInterval(sync, 60000);
+setInterval(sync, 60000);
+
+setTimeout(sync, 1000);

services/utils.js

@@ -7,7 +7,7 @@ function randomToken(length) {
 }
 
 function newNoteId() {
-    return randomString(32, '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ');
+    return randomString(22, '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ');
 }
 
 function randomString(length, chars) {