Generate ACME certs

acme.js

@@ -0,0 +1,118 @@
+'use strict';
+
+const config = require('wild-config');
+const restify = require('restify');
+const log = require('npmlog');
+const logger = require('restify-logger');
+const db = require('./lib/db');
+const Gelf = require('gelf');
+const os = require('os');
+
+const acmeRoutes = require('./lib/api/acme');
+
+let loggelf;
+
+const serverOptions = {
+    name: 'WildDuck ACME Agent',
+    strictRouting: true,
+    maxParamLength: 196
+};
+
+const server = restify.createServer(serverOptions);
+
+server.use(restify.plugins.gzipResponse());
+
+server.use(
+    restify.plugins.queryParser({
+        allowDots: true,
+        mapParams: true
+    })
+);
+
+logger.token('user-ip', req => ((req.params && req.params.ip) || '').toString().substr(0, 40) || '-');
+logger.token('user-sess', req => (req.params && req.params.sess) || '-');
+
+logger.token('user', req => (req.user && req.user.toString()) || '-');
+logger.token('url', req => {
+    if (/\baccessToken=/.test(req.url)) {
+        return req.url.replace(/\baccessToken=[^&]+/g, 'accessToken=' + 'x'.repeat(6));
+    }
+    return req.url;
+});
+
+server.use(
+    logger(':remote-addr :user [:user-ip/:user-sess] :method :url :status :time-spent :append', {
+        stream: {
+            write: message => {
+                message = (message || '').toString();
+                if (message) {
+                    log.http('ACME', message.replace('\n', '').trim());
+                }
+            }
+        }
+    })
+);
+
+module.exports = done => {
+    if (!config.acme.agent.enabled) {
+        return setImmediate(() => done(null, false));
+    }
+
+    let started = false;
+
+    const component = config.log.gelf.component || 'wildduck';
+    const hostname = config.log.gelf.hostname || os.hostname();
+    const gelf =
+        config.log.gelf && config.log.gelf.enabled
+            ? new Gelf(config.log.gelf.options)
+            : {
+                  // placeholder
+                  emit: (key, message) => log.info('Gelf', JSON.stringify(message))
+              };
+
+    loggelf = message => {
+        if (typeof message === 'string') {
+            message = {
+                short_message: message
+            };
+        }
+        message = message || {};
+
+        if (!message.short_message || message.short_message.indexOf(component.toUpperCase()) !== 0) {
+            message.short_message = component.toUpperCase() + ' ' + (message.short_message || '');
+        }
+
+        message.facility = component; // facility is deprecated but set by the driver if not provided
+        message.host = hostname;
+        message.timestamp = Date.now() / 1000;
+        message._component = component;
+        Object.keys(message).forEach(key => {
+            if (!message[key]) {
+                delete message[key];
+            }
+        });
+        gelf.emit('gelf.log', message);
+    };
+
+    server.loggelf = message => loggelf(message);
+
+    acmeRoutes(db, server);
+
+    server.on('error', err => {
+        if (!started) {
+            started = true;
+            return done(err);
+        }
+
+        log.error('ACME', err);
+    });
+
+    server.listen(config.acme.agent.port, config.acme.agent.host, () => {
+        if (started) {
+            return server.close();
+        }
+        started = true;
+        log.info('ACME', 'Server listening on %s:%s', config.acme.agent.host || '0.0.0.0', config.acme.agent.port);
+        done(null, server);
+    });
+};

config/acme.toml

@@ -1,6 +1,6 @@
 
 # if hostname has a CAA record set then match it against this list
-caaDomains = ["letsencrypt.org"]
+caaDomains = [ "letsencrypt.org" ]
 
 keyBits = 2048
 keyExponent = 65537
@@ -14,3 +14,10 @@ email = "domainadmin@example.com" # must be valid email address
 #key = "production"
 #directoryUrl = "https://acme-v02.api.letsencrypt.org/directory"
 #email = "domainadmin@example.com" # must be valid email address
+
+[agent]
+# If enabled then starts a HTTP server that listens for ACME verification requests
+# If you have API already listening on port 80 then you don't need this
+enabled = true
+port = 7003 # use 80 in production
+redirect = "https://wildduck.email" # redirect requests unrelated to ACME updates to this URL

config/api.toml

@@ -1,5 +1,5 @@
 enabled=true
-port=7003
+port=8080
 # by default bind to localhost only
 host="127.0.0.1"
 

lib/acme/acme-challenge.js

@@ -41,7 +41,7 @@ class AcmeChallenge {
                     '_acme.secret.expires': new Date(Date.now() + this.ttl)
                 }
             },
-            { returnOriginal: false }
+            { returnDocument: 'after' }
         );
 
         if (!domainData || !domainData.value) {

lib/acme/certs.js

@@ -21,6 +21,7 @@ if (config.resolver && config.resolver.ns && config.resolver.ns.length) {
     resolver.setServers([].concat(config.resolver.ns || []));
 }
 
+const RENEW_AFTER_REMAINING = 10000 + 30 * 24 * 3600 * 1000;
 const BLOCK_RENEW_AFTER_ERROR_TTL = 10; //3600;
 
 const acme = ACME.create({
@@ -197,7 +198,7 @@ const acquireCert = async (domain, acmeOptions, certificateData) => {
     try {
         // reload from db, maybe already renewed
         certificateData = await certHandler.getRecord({ _id: certificateData._id }, true);
-        if (certificateData.expires > new Date(Date.now() + 10000 + 30 * 24 * 3600 * 1000)) {
+        if (certificateData.expires > new Date(Date.now() + RENEW_AFTER_REMAINING)) {
             // no need to renew
             return certificateData;
         }
@@ -250,11 +251,11 @@ const acquireCert = async (domain, acmeOptions, certificateData) => {
 
         let updates = {
             cert: cert.cert,
-            ca: cert.chain,
+            ca: [].concat(cert.chain || []),
             validFrom: new Date(parsed.validFrom),
             expires: new Date(parsed.validTo),
             altNames: parsed.dnsNames,
-            issuer: parsed.issuer.CN,
+            issuer: parsed.issuer.commonName,
             lastCheck: now,
             status: 'valid'
         };

lib/api/acme.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const config = require('wild-config');
 const log = require('npmlog');
 const Joi = require('joi');
 const AcmeChallenge = require('../acme/acme-challenge');
@@ -67,4 +68,8 @@ module.exports = (db, server) => {
             res.end(challenge.keyAuthorization);
         })
     );
+
+    server.on('NotFound', (req, res, err, cb) => {
+        res.redirect(302, config.acme.agent.redirect, cb);
+    });
 };

lib/api/certs.js

@@ -280,8 +280,15 @@ module.exports = (db, server) => {
             }
 
             if (result.value.acme) {
-                // TODO: push to cert renewal queue
+                let now = new Date();
-                await getCertificate(result.value.servername, config.acme);
+                await db.database.collection('tasks').insertOne({
+                    task: 'acme',
+                    locked: false,
+                    lockedUntil: now,
+                    created: now,
+                    status: 'queued',
+                    servername: result.value.servername
+                });
             }
 
             res.json(response);

lib/api/domainaccess.js

@@ -67,7 +67,7 @@ module.exports = (db, server) => {
                     {
                         upsert: true,
                         projection: { _id: true },
-                        returnOriginal: false
+                        returnDocument: 'after'
                     }
                 );
             } catch (err) {

lib/api/messages.js

@@ -1473,7 +1473,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
                             updated: now
                         }
                     },
-                    { upsert: true, returnOriginal: false }
+                    { upsert: true, returnDocument: 'after' }
                 );
 
                 res.json({
@@ -2347,7 +2347,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => {
                                 }
                             },
                             {
-                                returnOriginal: false,
+                                returnDocument: 'after',
                                 projection: {
                                     uid: true,
                                     flags: true

lib/api/submit.js

@@ -126,7 +126,7 @@ module.exports = (db, server, messageHandler, userHandler) => {
                                 $addToSet
                             },
                             {
-                                returnOriginal: false,
+                                returnDocument: 'after',
                                 projection: {
                                     'mimeTree.parsedHeader': true,
                                     uid: true,

lib/api/users.js

@@ -1198,7 +1198,7 @@ module.exports = (db, server, userHandler) => {
                         }
                     },
                     {
-                        returnOriginal: true,
+                        returnDocument: 'before',
                         projection: {
                             storageUsed: true
                         }

lib/attachments/gridstore-storage.js

@@ -166,7 +166,7 @@ class GridstoreStorage {
                     }
                 },
                 {
-                    returnOriginal: false
+                    returnDocument: 'after'
                 },
                 (err, result) => {
                     if (err) {
@@ -370,7 +370,7 @@ class GridstoreStorage {
                 }
             },
             {
-                returnOriginal: false
+                returnDocument: 'after'
             },
             (err, result) => {
                 if (err) {

lib/cert-handler.js

@@ -128,7 +128,7 @@ class CertHandler {
         try {
             r = await this.database.collection('certs').findOneAndUpdate(query, changes, {
                 upsert: false,
-                returnOriginal: false
+                returnDocument: 'after'
             });
         } catch (err) {
             if (err) {
@@ -145,13 +145,13 @@ class CertHandler {
             throw err;
         }
 
-        if (this.redis && updates.privateKey) {
+        if (this.redis && updates.cert) {
             try {
                 await publish(this.redis, {
                     ev: CERT_UPDATED,
                     cert: r.value._id.toString(),
                     servername: r.value.servername,
-                    fingerprint: fp
+                    fingerprint: r.value.fp
                 });
             } catch (err) {
                 // ignore?
@@ -192,7 +192,7 @@ class CertHandler {
                 },
                 {
                     upsert: false,
-                    returnOriginal: false
+                    returnDocument: 'after'
                 }
             );
         } catch (err) {
@@ -210,19 +210,6 @@ class CertHandler {
             throw err;
         }
 
-        if (this.redis) {
-            try {
-                await publish(this.redis, {
-                    ev: CERT_UPDATED,
-                    cert: r.value._id.toString(),
-                    servername: r.value.servername,
-                    fingerprint: fp
-                });
-            } catch (err) {
-                // ignore?
-            }
-        }
-
         return privateKey;
     }
 
@@ -265,7 +252,7 @@ class CertHandler {
             certData.cert = cert;
         }
 
-        certData.ca = ca;
+        certData.ca = [].concat(ca || []);
 
         if (primaryCert) {
             try {
@@ -323,7 +310,7 @@ class CertHandler {
                 { $set: certData, $inc: { v: 1 }, $setOnInsert: { servername, created: new Date() } },
                 {
                     upsert: true,
-                    returnOriginal: false
+                    returnDocument: 'after'
                 }
             );
         } catch (err) {

lib/dkim-handler.js

@@ -172,7 +172,7 @@ class DkimHandler {
                     dkimData,
                     {
                         upsert: true,
-                        returnOriginal: false
+                        returnDocument: 'after'
                     },
                     (err, r) => {
                         if (err) {

lib/handlers/on-copy.js

@@ -90,7 +90,7 @@ async function copyHandler(server, messageHandler, connection, mailbox, update,
                     }
                 },
                 {
-                    returnOriginal: false,
+                    returnDocument: 'after',
                     projection: {
                         storageUsed: true
                     },
@@ -157,7 +157,7 @@ async function copyHandler(server, messageHandler, connection, mailbox, update,
                         uidNext: true,
                         modifyIndex: true
                     },
-                    returnOriginal: true,
+                    returnDocument: 'before',
                     maxTimeMS: consts.DB_MAX_TIME_MAILBOXES
                 }
             );

lib/handlers/on-store.js

@@ -52,7 +52,7 @@ module.exports = server => (mailbox, update, session, callback) => {
                         }
                     },
                     {
-                        returnOriginal: false,
+                        returnDocument: 'after',
                         maxTimeMS: consts.DB_MAX_TIME_MAILBOXES
                     },
                     (err, item) => {

lib/imap-notifier.js

@@ -167,7 +167,7 @@ class ImapNotifier extends EventEmitter {
                         }
                     },
                     {
-                        returnOriginal: false
+                        returnDocument: 'after'
                     },
                     (err, item) => {
                         if (err) {

lib/message-handler.js

@@ -370,7 +370,7 @@ class MessageHandler {
                             }
                         },
                         {
-                            returnOriginal: false,
+                            returnDocument: 'after',
                             projection: {
                                 storageUsed: true
                             }
@@ -403,7 +403,7 @@ class MessageHandler {
                                         }
                                     },
                                     {
-                                        returnOriginal: false,
+                                        returnDocument: 'after',
                                         projection: {
                                             storageUsed: true
                                         }
@@ -446,7 +446,7 @@ class MessageHandler {
                                 },
                                 {
                                     // use original value to get correct UIDNext
-                                    returnOriginal: true
+                                    returnDocument: 'before'
                                 },
                                 (err, item) => {
                                     if (err) {
@@ -628,7 +628,7 @@ class MessageHandler {
                 }
             },
             {
-                returnOriginal: false,
+                returnDocument: 'after',
                 projection: {
                     storageUsed: true
                 }
@@ -812,7 +812,7 @@ class MessageHandler {
                         }
                     },
                     {
-                        returnOriginal: false,
+                        returnDocument: 'after',
                         projection: {
                             _id: true,
                             uidNext: true,
@@ -915,7 +915,7 @@ class MessageHandler {
                                             uidNext: true,
                                             modifyIndex: true
                                         },
-                                        returnOriginal: true
+                                        returnDocument: 'before'
                                     },
                                     (err, item) => {
                                         if (err) {
@@ -1389,7 +1389,7 @@ class MessageHandler {
                 }
             },
             {
-                returnOriginal: false
+                returnDocument: 'after'
             },
             (err, r) => {
                 if (err) {
@@ -1535,7 +1535,7 @@ class MessageHandler {
                 }
             },
             {
-                returnOriginal: false
+                returnDocument: 'after'
             },
             (err, item) => {
                 if (err) {
@@ -1604,7 +1604,7 @@ class MessageHandler {
                                     uid: true,
                                     flags: true
                                 },
-                                returnOriginal: false
+                                returnDocument: 'after'
                             },
                             (err, item) => {
                                 if (err) {

lib/tasks/acme.js

@@ -0,0 +1,19 @@
+'use strict';
+
+const log = require('npmlog');
+const config = require('wild-config');
+
+let run = async (taskData, options) => {
+    const { getCertificate } = options;
+
+    let cert = await getCertificate(taskData.servername, config.acme);
+
+    log.verbose('Tasks', 'task=acme id=%s servername=%s status=%s', taskData._id, taskData.servername, cert && cert.status);
+    return true;
+};
+
+module.exports = (taskData, options, callback) => {
+    run(taskData, options)
+        .then(response => callback(null, response))
+        .catch(callback);
+};

lib/tasks/audit.js

@@ -4,8 +4,7 @@ const log = require('npmlog');
 const db = require('../db');
 
 let run = async (taskData, options) => {
-    const messageHandler = options.messageHandler;
+    const { auditHandler, messageHandler } = options;
-    const auditHandler = options.auditHandler;
 
     let query = {
         user: taskData.user

lib/tasks/quota.js

@@ -66,7 +66,7 @@ module.exports = (taskData, options, callback) => {
                             }
                         },
                         {
-                            returnOriginal: true,
+                            returnDocument: 'before',
                             projection: {
                                 storageUsed: true
                             }

lib/user-handler.js

@@ -1610,7 +1610,7 @@ class UserHandler {
                     $set: updates
                 },
                 {
-                    returnOriginal: false,
+                    returnDocument: 'after',
                     maxTimeMS: consts.DB_MAX_TIME_USERS
                 }
             );
@@ -3234,7 +3234,7 @@ class UserHandler {
                 },
                 updateQuery,
                 {
-                    returnOriginal: false,
+                    returnDocument: 'after',
                     maxTimeMS: consts.DB_MAX_TIME_USERS
                 }
             );
@@ -3388,7 +3388,7 @@ class UserHandler {
             {
                 upsert: true,
                 projection: { _id: true },
-                returnOriginal: false,
+                returnDocument: 'after',
                 maxTimeMS: consts.DB_MAX_TIME_USERS
             }
         );
@@ -3502,7 +3502,7 @@ class UserHandler {
                             lockedUntil: deleteAfter
                         }
                     },
-                    { returnOriginal: false }
+                    { returnDocument: 'after' }
                 );
 
                 if (r && r.value) {

package.json

@@ -31,7 +31,7 @@
         "grunt-mocha-test": "0.13.3",
         "grunt-shell-spawn": "0.4.0",
         "grunt-wait": "0.3.0",
-        "imapflow": "1.0.58",
+        "imapflow": "1.0.59",
         "mailparser": "3.2.0",
         "mocha": "9.0.0",
         "request": "2.88.2",
@@ -55,7 +55,7 @@
         "humanname": "0.2.2",
         "iconv-lite": "0.6.3",
         "ioredfour": "1.0.2-ioredis-03",
-        "ioredis": "4.27.5",
+        "ioredis": "4.27.6",
         "ipaddr": "0.1.0",
         "ipaddr.js": "2.0.1",
         "isemail": "3.2.0",

pop3.js

@@ -342,7 +342,7 @@ function markAsSeen(session, messages, callback) {
             }
         },
         {
-            returnOriginal: false
+            returnDocument: 'after'
         },
         (err, item) => {
             if (err) {

tasks.js

@@ -9,7 +9,11 @@ const yaml = require('js-yaml');
 const fs = require('fs');
 const MessageHandler = require('./lib/message-handler');
 const MailboxHandler = require('./lib/mailbox-handler');
+const CertHandler = require('./lib/cert-handler');
 const AuditHandler = require('./lib/audit-handler');
+
+const { getCertificate } = require('./lib/acme/certs');
+
 const setupIndexes = yaml.load(fs.readFileSync(__dirname + '/indexes.yaml', 'utf8'));
 const Gelf = require('gelf');
 const os = require('os');
@@ -18,11 +22,13 @@ const taskRestore = require('./lib/tasks/restore');
 const taskUserDelete = require('./lib/tasks/user-delete');
 const taskQuota = require('./lib/tasks/quota');
 const taskAudit = require('./lib/tasks/audit');
+const taskAcme = require('./lib/tasks/acme');
 const taskClearFolder = require('./lib/tasks/clear-folder');
 
 let messageHandler;
 let mailboxHandler;
 let auditHandler;
+let certHandler;
 let gcTimeout;
 let taskTimeout;
 let gcLock;
@@ -102,6 +108,13 @@ module.exports.start = callback => {
         loggelf: message => loggelf(message)
     });
 
+    certHandler = new CertHandler({
+        cipher: config.certs && config.certs.cipher,
+        secret: config.certs && config.certs.secret,
+        database: db.database,
+        redis: db.redis
+    });
+
     let start = () => {
         // setup ready
 
@@ -454,7 +467,7 @@ function runTasks() {
                         }
                     },
                     {
-                        returnOriginal: false
+                        returnDocument: 'after'
                     },
                     (err, r) => {
                         if (err) {
@@ -605,6 +618,23 @@ function processTask(taskData, callback) {
                 }
             );
 
+        case 'acme':
+            return taskAcme(
+                taskData,
+                {
+                    certHandler,
+                    getCertificate,
+                    loggelf
+                },
+                err => {
+                    if (err) {
+                        return callback(err);
+                    }
+                    // release
+                    callback(null, true);
+                }
+            );
+
         case 'clear-folder':
             return taskClearFolder(
                 taskData,

worker.js

@@ -6,6 +6,7 @@ const imap = require('./imap');
 const pop3 = require('./pop3');
 const lmtp = require('./lmtp');
 const api = require('./api');
+const acme = require('./acme');
 const tasks = require('./tasks');
 const webhooks = require('./webhooks');
 const plugins = require('./lib/plugins');
@@ -67,6 +68,14 @@ db.connect(err => {
                                 return setTimeout(() => process.exit(1), 3000);
                             }
 
+                            // Start HTTP ACME server
+                            acme(err => {
+                                if (err) {
+                                    log.error('App', 'Failed to start ACME server');
+                                    errors.notify(err);
+                                    return setTimeout(() => process.exit(1), 3000);
+                                }
+
                                 // downgrade user and group if needed
                                 if (config.group) {
                                     try {
@@ -106,4 +115,5 @@ db.connect(err => {
                 });
             });
         });
+    });
 });