From 9ae177869eb1e65177d3c88cc1d60540e05dd7b4 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Tue, 15 Jun 2021 10:47:18 +0300 Subject: [PATCH] Generate ACME certs --- acme.js | 118 +++++++++++++++++++++++++++ config/acme.toml | 11 ++- config/api.toml | 2 +- lib/acme/acme-challenge.js | 2 +- lib/acme/certs.js | 7 +- lib/api/acme.js | 5 ++ lib/api/certs.js | 11 ++- lib/api/domainaccess.js | 2 +- lib/api/messages.js | 4 +- lib/api/submit.js | 2 +- lib/api/users.js | 2 +- lib/attachments/gridstore-storage.js | 4 +- lib/cert-handler.js | 25 ++---- lib/dkim-handler.js | 2 +- lib/handlers/on-copy.js | 4 +- lib/handlers/on-store.js | 2 +- lib/imap-notifier.js | 2 +- lib/message-handler.js | 18 ++-- lib/tasks/acme.js | 19 +++++ lib/tasks/audit.js | 3 +- lib/tasks/quota.js | 2 +- lib/user-handler.js | 8 +- package.json | 4 +- pop3.js | 2 +- tasks.js | 32 +++++++- worker.js | 62 ++++++++------ 26 files changed, 269 insertions(+), 86 deletions(-) create mode 100644 acme.js create mode 100644 lib/tasks/acme.js diff --git a/acme.js b/acme.js new file mode 100644 index 00000000..d99d694d --- /dev/null +++ b/acme.js @@ -0,0 +1,118 @@ +'use strict'; + +const config = require('wild-config'); +const restify = require('restify'); +const log = require('npmlog'); +const logger = require('restify-logger'); +const db = require('./lib/db'); +const Gelf = require('gelf'); +const os = require('os'); + +const acmeRoutes = require('./lib/api/acme'); + +let loggelf; + +const serverOptions = { + name: 'WildDuck ACME Agent', + strictRouting: true, + maxParamLength: 196 +}; + +const server = restify.createServer(serverOptions); + +server.use(restify.plugins.gzipResponse()); + +server.use( + restify.plugins.queryParser({ + allowDots: true, + mapParams: true + }) +); + +logger.token('user-ip', req => ((req.params && req.params.ip) || '').toString().substr(0, 40) || '-'); +logger.token('user-sess', req => (req.params && req.params.sess) || '-'); + +logger.token('user', req => (req.user && req.user.toString()) || '-'); +logger.token('url', req => { + if (/\baccessToken=/.test(req.url)) { + return req.url.replace(/\baccessToken=[^&]+/g, 'accessToken=' + 'x'.repeat(6)); + } + return req.url; +}); + +server.use( + logger(':remote-addr :user [:user-ip/:user-sess] :method :url :status :time-spent :append', { + stream: { + write: message => { + message = (message || '').toString(); + if (message) { + log.http('ACME', message.replace('\n', '').trim()); + } + } + } + }) +); + +module.exports = done => { + if (!config.acme.agent.enabled) { + return setImmediate(() => done(null, false)); + } + + let started = false; + + const component = config.log.gelf.component || 'wildduck'; + const hostname = config.log.gelf.hostname || os.hostname(); + const gelf = + config.log.gelf && config.log.gelf.enabled + ? new Gelf(config.log.gelf.options) + : { + // placeholder + emit: (key, message) => log.info('Gelf', JSON.stringify(message)) + }; + + loggelf = message => { + if (typeof message === 'string') { + message = { + short_message: message + }; + } + message = message || {}; + + if (!message.short_message || message.short_message.indexOf(component.toUpperCase()) !== 0) { + message.short_message = component.toUpperCase() + ' ' + (message.short_message || ''); + } + + message.facility = component; // facility is deprecated but set by the driver if not provided + message.host = hostname; + message.timestamp = Date.now() / 1000; + message._component = component; + Object.keys(message).forEach(key => { + if (!message[key]) { + delete message[key]; + } + }); + gelf.emit('gelf.log', message); + }; + + server.loggelf = message => loggelf(message); + + acmeRoutes(db, server); + + server.on('error', err => { + if (!started) { + started = true; + return done(err); + } + + log.error('ACME', err); + }); + + server.listen(config.acme.agent.port, config.acme.agent.host, () => { + if (started) { + return server.close(); + } + started = true; + log.info('ACME', 'Server listening on %s:%s', config.acme.agent.host || '0.0.0.0', config.acme.agent.port); + done(null, server); + }); +}; diff --git a/config/acme.toml b/config/acme.toml index 6c129211..8bdff114 100644 --- a/config/acme.toml +++ b/config/acme.toml @@ -1,6 +1,6 @@ # if hostname has a CAA record set then match it against this list -caaDomains = ["letsencrypt.org"] +caaDomains = [ "letsencrypt.org" ] keyBits = 2048 keyExponent = 65537 @@ -13,4 +13,11 @@ email = "domainadmin@example.com" # must be valid email address # ACME production settings #key = "production" #directoryUrl = "https://acme-v02.api.letsencrypt.org/directory" -#email = "domainadmin@example.com" # must be valid email address \ No newline at end of file +#email = "domainadmin@example.com" # must be valid email address + +[agent] +# If enabled then starts a HTTP server that listens for ACME verification requests +# If you have API already listening on port 80 then you don't need this +enabled = true +port = 7003 # use 80 in production +redirect = "https://wildduck.email" # redirect requests unrelated to ACME updates to this URL diff --git a/config/api.toml b/config/api.toml index 6c6dccdb..740ff3ef 100644 --- a/config/api.toml +++ b/config/api.toml @@ -1,5 +1,5 @@ enabled=true -port=7003 +port=8080 # by default bind to localhost only host="127.0.0.1" diff --git a/lib/acme/acme-challenge.js b/lib/acme/acme-challenge.js index ae3658fb..04d2c1fd 100644 --- a/lib/acme/acme-challenge.js +++ b/lib/acme/acme-challenge.js @@ -41,7 +41,7 @@ class AcmeChallenge { '_acme.secret.expires': new Date(Date.now() + this.ttl) } }, - { returnOriginal: false } + { returnDocument: 'after' } ); if (!domainData || !domainData.value) { diff --git a/lib/acme/certs.js b/lib/acme/certs.js index abb05924..5c772723 100644 --- a/lib/acme/certs.js +++ b/lib/acme/certs.js @@ -21,6 +21,7 @@ if (config.resolver && config.resolver.ns && config.resolver.ns.length) { resolver.setServers([].concat(config.resolver.ns || [])); } +const RENEW_AFTER_REMAINING = 10000 + 30 * 24 * 3600 * 1000; const BLOCK_RENEW_AFTER_ERROR_TTL = 10; //3600; const acme = ACME.create({ @@ -197,7 +198,7 @@ const acquireCert = async (domain, acmeOptions, certificateData) => { try { // reload from db, maybe already renewed certificateData = await certHandler.getRecord({ _id: certificateData._id }, true); - if (certificateData.expires > new Date(Date.now() + 10000 + 30 * 24 * 3600 * 1000)) { + if (certificateData.expires > new Date(Date.now() + RENEW_AFTER_REMAINING)) { // no need to renew return certificateData; } @@ -250,11 +251,11 @@ const acquireCert = async (domain, acmeOptions, certificateData) => { let updates = { cert: cert.cert, - ca: cert.chain, + ca: [].concat(cert.chain || []), validFrom: new Date(parsed.validFrom), expires: new Date(parsed.validTo), altNames: parsed.dnsNames, - issuer: parsed.issuer.CN, + issuer: parsed.issuer.commonName, lastCheck: now, status: 'valid' }; diff --git a/lib/api/acme.js b/lib/api/acme.js index 3d185b62..00dcbfb2 100644 --- a/lib/api/acme.js +++ b/lib/api/acme.js @@ -1,5 +1,6 @@ 'use strict'; +const config = require('wild-config'); const log = require('npmlog'); const Joi = require('joi'); const AcmeChallenge = require('../acme/acme-challenge'); @@ -67,4 +68,8 @@ module.exports = (db, server) => { res.end(challenge.keyAuthorization); }) ); + + server.on('NotFound', (req, res, err, cb) => { + res.redirect(302, config.acme.agent.redirect, cb); + }); }; diff --git a/lib/api/certs.js b/lib/api/certs.js index 130a54c9..53dfaafa 100644 --- a/lib/api/certs.js +++ b/lib/api/certs.js @@ -280,8 +280,15 @@ module.exports = (db, server) => { } if (result.value.acme) { - // TODO: push to cert renewal queue - await getCertificate(result.value.servername, config.acme); + let now = new Date(); + await db.database.collection('tasks').insertOne({ + task: 'acme', + locked: false, + lockedUntil: now, + created: now, + status: 'queued', + servername: result.value.servername + }); } res.json(response); diff --git a/lib/api/domainaccess.js b/lib/api/domainaccess.js index 5a175662..47d3b48b 100644 --- a/lib/api/domainaccess.js +++ b/lib/api/domainaccess.js @@ -67,7 +67,7 @@ module.exports = (db, server) => { { upsert: true, projection: { _id: true }, - returnOriginal: false + returnDocument: 'after' } ); } catch (err) { diff --git a/lib/api/messages.js b/lib/api/messages.js index 526dd060..4934c369 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -1473,7 +1473,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => { updated: now } }, - { upsert: true, returnOriginal: false } + { upsert: true, returnDocument: 'after' } ); res.json({ @@ -2347,7 +2347,7 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler) => { } }, { - returnOriginal: false, + returnDocument: 'after', projection: { uid: true, flags: true diff --git a/lib/api/submit.js b/lib/api/submit.js index c6c2fbea..b1fb283f 100644 --- a/lib/api/submit.js +++ b/lib/api/submit.js @@ -126,7 +126,7 @@ module.exports = (db, server, messageHandler, userHandler) => { $addToSet }, { - returnOriginal: false, + returnDocument: 'after', projection: { 'mimeTree.parsedHeader': true, uid: true, diff --git a/lib/api/users.js b/lib/api/users.js index ac59de42..4a334644 100644 --- a/lib/api/users.js +++ b/lib/api/users.js @@ -1198,7 +1198,7 @@ module.exports = (db, server, userHandler) => { } }, { - returnOriginal: true, + returnDocument: 'before', projection: { storageUsed: true } diff --git a/lib/attachments/gridstore-storage.js b/lib/attachments/gridstore-storage.js index 7ec98540..9271684f 100644 --- a/lib/attachments/gridstore-storage.js +++ b/lib/attachments/gridstore-storage.js @@ -166,7 +166,7 @@ class GridstoreStorage { } }, { - returnOriginal: false + returnDocument: 'after' }, (err, result) => { if (err) { @@ -370,7 +370,7 @@ class GridstoreStorage { } }, { - returnOriginal: false + returnDocument: 'after' }, (err, result) => { if (err) { diff --git a/lib/cert-handler.js b/lib/cert-handler.js index b1b4a0fb..fed1f0cf 100644 --- a/lib/cert-handler.js +++ b/lib/cert-handler.js @@ -128,7 +128,7 @@ class CertHandler { try { r = await this.database.collection('certs').findOneAndUpdate(query, changes, { upsert: false, - returnOriginal: false + returnDocument: 'after' }); } catch (err) { if (err) { @@ -145,13 +145,13 @@ class CertHandler { throw err; } - if (this.redis && updates.privateKey) { + if (this.redis && updates.cert) { try { await publish(this.redis, { ev: CERT_UPDATED, cert: r.value._id.toString(), servername: r.value.servername, - fingerprint: fp + fingerprint: r.value.fp }); } catch (err) { // ignore? @@ -192,7 +192,7 @@ class CertHandler { }, { upsert: false, - returnOriginal: false + returnDocument: 'after' } ); } catch (err) { @@ -210,19 +210,6 @@ class CertHandler { throw err; } - if (this.redis) { - try { - await publish(this.redis, { - ev: CERT_UPDATED, - cert: r.value._id.toString(), - servername: r.value.servername, - fingerprint: fp - }); - } catch (err) { - // ignore? - } - } - return privateKey; } @@ -265,7 +252,7 @@ class CertHandler { certData.cert = cert; } - certData.ca = ca; + certData.ca = [].concat(ca || []); if (primaryCert) { try { @@ -323,7 +310,7 @@ class CertHandler { { $set: certData, $inc: { v: 1 }, $setOnInsert: { servername, created: new Date() } }, { upsert: true, - returnOriginal: false + returnDocument: 'after' } ); } catch (err) { diff --git a/lib/dkim-handler.js b/lib/dkim-handler.js index ecc4b5d8..9911214e 100644 --- a/lib/dkim-handler.js +++ b/lib/dkim-handler.js @@ -172,7 +172,7 @@ class DkimHandler { dkimData, { upsert: true, - returnOriginal: false + returnDocument: 'after' }, (err, r) => { if (err) { diff --git a/lib/handlers/on-copy.js b/lib/handlers/on-copy.js index 58205b84..7445e8b2 100644 --- a/lib/handlers/on-copy.js +++ b/lib/handlers/on-copy.js @@ -90,7 +90,7 @@ async function copyHandler(server, messageHandler, connection, mailbox, update, } }, { - returnOriginal: false, + returnDocument: 'after', projection: { storageUsed: true }, @@ -157,7 +157,7 @@ async function copyHandler(server, messageHandler, connection, mailbox, update, uidNext: true, modifyIndex: true }, - returnOriginal: true, + returnDocument: 'before', maxTimeMS: consts.DB_MAX_TIME_MAILBOXES } ); diff --git a/lib/handlers/on-store.js b/lib/handlers/on-store.js index 7849ab1d..b541ca81 100644 --- a/lib/handlers/on-store.js +++ b/lib/handlers/on-store.js @@ -52,7 +52,7 @@ module.exports = server => (mailbox, update, session, callback) => { } }, { - returnOriginal: false, + returnDocument: 'after', maxTimeMS: consts.DB_MAX_TIME_MAILBOXES }, (err, item) => { diff --git a/lib/imap-notifier.js b/lib/imap-notifier.js index 1f693420..7c99c9c7 100644 --- a/lib/imap-notifier.js +++ b/lib/imap-notifier.js @@ -167,7 +167,7 @@ class ImapNotifier extends EventEmitter { } }, { - returnOriginal: false + returnDocument: 'after' }, (err, item) => { if (err) { diff --git a/lib/message-handler.js b/lib/message-handler.js index 240bb2a9..329e7fa2 100644 --- a/lib/message-handler.js +++ b/lib/message-handler.js @@ -370,7 +370,7 @@ class MessageHandler { } }, { - returnOriginal: false, + returnDocument: 'after', projection: { storageUsed: true } @@ -403,7 +403,7 @@ class MessageHandler { } }, { - returnOriginal: false, + returnDocument: 'after', projection: { storageUsed: true } @@ -446,7 +446,7 @@ class MessageHandler { }, { // use original value to get correct UIDNext - returnOriginal: true + returnDocument: 'before' }, (err, item) => { if (err) { @@ -628,7 +628,7 @@ class MessageHandler { } }, { - returnOriginal: false, + returnDocument: 'after', projection: { storageUsed: true } @@ -812,7 +812,7 @@ class MessageHandler { } }, { - returnOriginal: false, + returnDocument: 'after', projection: { _id: true, uidNext: true, @@ -915,7 +915,7 @@ class MessageHandler { uidNext: true, modifyIndex: true }, - returnOriginal: true + returnDocument: 'before' }, (err, item) => { if (err) { @@ -1389,7 +1389,7 @@ class MessageHandler { } }, { - returnOriginal: false + returnDocument: 'after' }, (err, r) => { if (err) { @@ -1535,7 +1535,7 @@ class MessageHandler { } }, { - returnOriginal: false + returnDocument: 'after' }, (err, item) => { if (err) { @@ -1604,7 +1604,7 @@ class MessageHandler { uid: true, flags: true }, - returnOriginal: false + returnDocument: 'after' }, (err, item) => { if (err) { diff --git a/lib/tasks/acme.js b/lib/tasks/acme.js new file mode 100644 index 00000000..ccaa21bd --- /dev/null +++ b/lib/tasks/acme.js @@ -0,0 +1,19 @@ +'use strict'; + +const log = require('npmlog'); +const config = require('wild-config'); + +let run = async (taskData, options) => { + const { getCertificate } = options; + + let cert = await getCertificate(taskData.servername, config.acme); + + log.verbose('Tasks', 'task=acme id=%s servername=%s status=%s', taskData._id, taskData.servername, cert && cert.status); + return true; +}; + +module.exports = (taskData, options, callback) => { + run(taskData, options) + .then(response => callback(null, response)) + .catch(callback); +}; diff --git a/lib/tasks/audit.js b/lib/tasks/audit.js index 96d0ec68..abbe7cb2 100644 --- a/lib/tasks/audit.js +++ b/lib/tasks/audit.js @@ -4,8 +4,7 @@ const log = require('npmlog'); const db = require('../db'); let run = async (taskData, options) => { - const messageHandler = options.messageHandler; - const auditHandler = options.auditHandler; + const { auditHandler, messageHandler } = options; let query = { user: taskData.user diff --git a/lib/tasks/quota.js b/lib/tasks/quota.js index 3fa696f0..4de1b15e 100644 --- a/lib/tasks/quota.js +++ b/lib/tasks/quota.js @@ -66,7 +66,7 @@ module.exports = (taskData, options, callback) => { } }, { - returnOriginal: true, + returnDocument: 'before', projection: { storageUsed: true } diff --git a/lib/user-handler.js b/lib/user-handler.js index 3653a9a8..c9b853d6 100644 --- a/lib/user-handler.js +++ b/lib/user-handler.js @@ -1610,7 +1610,7 @@ class UserHandler { $set: updates }, { - returnOriginal: false, + returnDocument: 'after', maxTimeMS: consts.DB_MAX_TIME_USERS } ); @@ -3234,7 +3234,7 @@ class UserHandler { }, updateQuery, { - returnOriginal: false, + returnDocument: 'after', maxTimeMS: consts.DB_MAX_TIME_USERS } ); @@ -3388,7 +3388,7 @@ class UserHandler { { upsert: true, projection: { _id: true }, - returnOriginal: false, + returnDocument: 'after', maxTimeMS: consts.DB_MAX_TIME_USERS } ); @@ -3502,7 +3502,7 @@ class UserHandler { lockedUntil: deleteAfter } }, - { returnOriginal: false } + { returnDocument: 'after' } ); if (r && r.value) { diff --git a/package.json b/package.json index 4a084de7..b5a188bb 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "grunt-mocha-test": "0.13.3", "grunt-shell-spawn": "0.4.0", "grunt-wait": "0.3.0", - "imapflow": "1.0.58", + "imapflow": "1.0.59", "mailparser": "3.2.0", "mocha": "9.0.0", "request": "2.88.2", @@ -55,7 +55,7 @@ "humanname": "0.2.2", "iconv-lite": "0.6.3", "ioredfour": "1.0.2-ioredis-03", - "ioredis": "4.27.5", + "ioredis": "4.27.6", "ipaddr": "0.1.0", "ipaddr.js": "2.0.1", "isemail": "3.2.0", diff --git a/pop3.js b/pop3.js index ed556493..3fdaa695 100644 --- a/pop3.js +++ b/pop3.js @@ -342,7 +342,7 @@ function markAsSeen(session, messages, callback) { } }, { - returnOriginal: false + returnDocument: 'after' }, (err, item) => { if (err) { diff --git a/tasks.js b/tasks.js index abdb2ba1..706d7629 100644 --- a/tasks.js +++ b/tasks.js @@ -9,7 +9,11 @@ const yaml = require('js-yaml'); const fs = require('fs'); const MessageHandler = require('./lib/message-handler'); const MailboxHandler = require('./lib/mailbox-handler'); +const CertHandler = require('./lib/cert-handler'); const AuditHandler = require('./lib/audit-handler'); + +const { getCertificate } = require('./lib/acme/certs'); + const setupIndexes = yaml.load(fs.readFileSync(__dirname + '/indexes.yaml', 'utf8')); const Gelf = require('gelf'); const os = require('os'); @@ -18,11 +22,13 @@ const taskRestore = require('./lib/tasks/restore'); const taskUserDelete = require('./lib/tasks/user-delete'); const taskQuota = require('./lib/tasks/quota'); const taskAudit = require('./lib/tasks/audit'); +const taskAcme = require('./lib/tasks/acme'); const taskClearFolder = require('./lib/tasks/clear-folder'); let messageHandler; let mailboxHandler; let auditHandler; +let certHandler; let gcTimeout; let taskTimeout; let gcLock; @@ -102,6 +108,13 @@ module.exports.start = callback => { loggelf: message => loggelf(message) }); + certHandler = new CertHandler({ + cipher: config.certs && config.certs.cipher, + secret: config.certs && config.certs.secret, + database: db.database, + redis: db.redis + }); + let start = () => { // setup ready @@ -454,7 +467,7 @@ function runTasks() { } }, { - returnOriginal: false + returnDocument: 'after' }, (err, r) => { if (err) { @@ -605,6 +618,23 @@ function processTask(taskData, callback) { } ); + case 'acme': + return taskAcme( + taskData, + { + certHandler, + getCertificate, + loggelf + }, + err => { + if (err) { + return callback(err); + } + // release + callback(null, true); + } + ); + case 'clear-folder': return taskClearFolder( taskData, diff --git a/worker.js b/worker.js index 8c4a6cea..90e8fcd8 100644 --- a/worker.js +++ b/worker.js @@ -6,6 +6,7 @@ const imap = require('./imap'); const pop3 = require('./pop3'); const lmtp = require('./lmtp'); const api = require('./api'); +const acme = require('./acme'); const tasks = require('./tasks'); const webhooks = require('./webhooks'); const plugins = require('./lib/plugins'); @@ -67,37 +68,46 @@ db.connect(err => { return setTimeout(() => process.exit(1), 3000); } - // downgrade user and group if needed - if (config.group) { - try { - process.setgid(config.group); - log.info('App', 'Changed group to "%s" (%s)', config.group, process.getgid()); - } catch (E) { - log.error('App', 'Failed to change group to "%s" (%s)', config.group, E.message); - errors.notify(E); - return setTimeout(() => process.exit(1), 3000); - } - } - if (config.user) { - try { - process.setuid(config.user); - log.info('App', 'Changed user to "%s" (%s)', config.user, process.getuid()); - } catch (E) { - log.error('App', 'Failed to change user to "%s" (%s)', config.user, E.message); - errors.notify(E); - return setTimeout(() => process.exit(1), 3000); - } - } - - plugins.init(err => { + // Start HTTP ACME server + acme(err => { if (err) { - log.error('App', 'Failed to start plugins'); + log.error('App', 'Failed to start ACME server'); errors.notify(err); return setTimeout(() => process.exit(1), 3000); } - plugins.runHooks('init', () => { - log.info('App', 'All servers started, ready to process some mail'); + // downgrade user and group if needed + if (config.group) { + try { + process.setgid(config.group); + log.info('App', 'Changed group to "%s" (%s)', config.group, process.getgid()); + } catch (E) { + log.error('App', 'Failed to change group to "%s" (%s)', config.group, E.message); + errors.notify(E); + return setTimeout(() => process.exit(1), 3000); + } + } + if (config.user) { + try { + process.setuid(config.user); + log.info('App', 'Changed user to "%s" (%s)', config.user, process.getuid()); + } catch (E) { + log.error('App', 'Failed to change user to "%s" (%s)', config.user, E.message); + errors.notify(E); + return setTimeout(() => process.exit(1), 3000); + } + } + + plugins.init(err => { + if (err) { + log.error('App', 'Failed to start plugins'); + errors.notify(err); + return setTimeout(() => process.exit(1), 3000); + } + + plugins.runHooks('init', () => { + log.info('App', 'All servers started, ready to process some mail'); + }); }); }); });