Use Redis to propagate state changes between different instances

README.md

@@ -2,7 +2,7 @@
 
 ![](https://cldup.com/qlZnwOz0na.jpg)
 
-This is a very early preview of an IMAP server built with Node.js and MongoDB.
+This is a very early preview of an IMAP server built with Node.js, MongoDB and Redis. Node.js runs the application, MongoDB is used as the mail store and Redis is used for ephemeral actions like publish/subscribe or caching.
 
 ### Goals of the Project
 
@@ -44,8 +44,12 @@ Not yet. These are some actions that must be done:
 1. Separate attachments from indexed mime tree and store these to GridFS. Currently entire message is loaded whenever a FETCH or SEARCH call is made (unless body needs not to be touched, for example if only FLAGs are checked). This also means that the message size is currently limited. MongoDB database records are capped at 16MB and this should contain also the metadata for the message.
 2. Optimize SEARCH queries to use MongoDB queries. Currently only simple stuff (flag, internaldate, not flag, modseq) is included in query and more complex comparisons are handled by the application but this means that too much data must be loaded from database (unless it is a very simple query like "SEARCH UNSEEN" that is already optimized)
 3. Optimize FETCH queries to load only partial data for BODY subparts
-4. Build a publish-subscribe solution to notify changes process-wide (and server-wide). Currently update notifications are propagated only inside the same process (journaled update data is available from DB for everybody but the notification that there is new data is not propagated outside current process).
-5. Parse incoming message into the mime tree as a stream. Currently the entire message is buffered in memory before being parsed.
+4. Parse incoming message into the mime tree as a stream. Currently the entire message is buffered in memory before being parsed.
+
+**What are the killer features?**
+
+1. Start as many instances as you want. You can start multiple Wild Duck instances in different machines and as long as they share the same MongoDB and Redis settings, users can connect to any instances. This is very different from more traditional IMAP servers where a single user always needs to connect (or proxied) to the same IMAP server. Wild Duck keeps all required state information in MongoDB, so it does not matter which IMAP instance you use.
+2. Super easy to tweak. The entire codebase is pure JavaScript, so there's nothing to compile or anything platform specific. If you need to tweak something then change the code, restart the app and you're ready to go. If it works on one machine then most probably it works in every other machine as well.
 
 ## Usage
 

config/default.js

@@ -6,6 +6,11 @@ module.exports = {
     },
 
     mongo: 'mongodb://127.0.0.1:27017/wildduck',
+    redis: {
+        host: 'localhost',
+        port: 6379,
+        db: 3
+    },
 
     imap: {
         port: 9993,

config/development.js

@@ -0,0 +1,37 @@
+'use strict';
+
+module.exports = {
+    log: {
+        level: 'silly'
+    },
+
+    mongo: 'mongodb://127.0.0.1:27017/wildduck',
+    redis: {
+        host: 'localhost',
+        port: 6379,
+        db: 3
+    },
+
+    imap: {
+        port: 9998,
+        host: '127.0.0.1'
+    },
+
+    lmtp: {
+        enabled: true,
+        port: 3424,
+        host: '0.0.0.0',
+        maxMB: 5
+    },
+
+    smtp: {
+        enabled: true,
+        port: 3525,
+        host: '0.0.0.0',
+        maxMB: 5
+    },
+
+    api: {
+        port: 8380
+    }
+};

imap-notifier.js

@@ -2,6 +2,7 @@
 
 const crypto = require('crypto');
 const EventEmitter = require('events').EventEmitter;
+const redis = require('redis');
 
 class ImapNotifier extends EventEmitter {
 
@@ -10,6 +11,9 @@ class ImapNotifier extends EventEmitter {
 
         this.database = options.database;
 
+        this.subsriber = redis.createClient();
+        this.publisher = redis.createClient();
+
         let logfunc = (...args) => {
             let level = args.shift() || 'DEBUG';
             let message = args.shift() || '';
@@ -26,7 +30,18 @@ class ImapNotifier extends EventEmitter {
         this._listeners = new EventEmitter();
         this._listeners.setMaxListeners(0);
 
-        EventEmitter.call(this);
+        this.publishTimer = false;
+        this.subsriber.on('message', (channel, message) => {
+            if (channel === 'wd_events') {
+                try {
+                    let data = JSON.parse(message);
+                    this._listeners.emit(data.e, data.p);
+                } catch (E) {
+                    //
+                }
+            }
+        });
+        this.subsriber.subscribe('wd_events');
     }
 
     /**
@@ -156,7 +171,11 @@ class ImapNotifier extends EventEmitter {
     fire(username, path, payload) {
         let eventName = this._eventName(username, path);
         setImmediate(() => {
-            this._listeners.emit(eventName, payload);
+            let data = JSON.stringify({
+                e: eventName,
+                p: payload
+            });
+            this.publisher.publish('wd_events', data);
         });
     }
 

package.json

@@ -17,7 +17,7 @@
     "grunt-eslint": "^19.0.0",
     "grunt-mocha-test": "^0.13.2",
     "mocha": "^3.2.0",
-    "nodemailer": "^3.1.4"
+    "nodemailer": "^3.1.5"
   },
   "dependencies": {
     "addressparser": "^1.0.1",