Added settings endpoint, allow to override ARCHIVE_TIME

api.js

@@ -44,6 +44,7 @@ const domainaliasRoutes = require('./lib/api/domainaliases');
 const dkimRoutes = require('./lib/api/dkim');
 const certsRoutes = require('./lib/api/certs');
 const webhooksRoutes = require('./lib/api/webhooks');
+const settingsRoutes = require('./lib/api/settings');
 
 let userHandler;
 let mailboxHandler;
@@ -527,6 +528,7 @@ module.exports = done => {
     dkimRoutes(db, server);
     certsRoutes(db, server);
     webhooksRoutes(db, server);
+    settingsRoutes(db, server);
 
     server.on('error', err => {
         if (!started) {

config/roles.json

@@ -108,6 +108,13 @@
             "read:any": ["*"],
             "update:any": ["*"],
             "delete:any": ["*"]
+        },
+
+        "settings": {
+            "create:any": ["*"],
+            "read:any": ["*"],
+            "update:any": ["*"],
+            "delete:any": ["*"]
         }
     },
 

indexes.yaml

@@ -708,8 +708,19 @@ indexes:
 
     - collection: settings
       index:
-          name: key_enumerable
+          name: key_unique
           unique: true
           key:
               key: 1
               enumerable: 1
+
+    - collection: settings
+      index:
+          name: enumerable_key
+          key:
+              enumerable: 1
+              key: 1
+
+deleteindexes:
+    - collection: settings
+      index: key_enumerable

lib/acme/certs.js

@@ -14,6 +14,7 @@ const { Resolver } = require('dns').promises;
 const resolver = new Resolver();
 const Joi = require('joi');
 const db = require('../db');
+const { SettingsHandler } = require('../settings-handler');
 
 if (config.resolver && config.resolver.ns && config.resolver.ns.length) {
     resolver.setServers([].concat(config.resolver.ns || []));
@@ -30,6 +31,8 @@ const acme = ACME.create({
     }
 });
 
+let settings;
+
 let getLock, releaseLock;
 
 // First try triggers initialization, others will wait until first is finished
@@ -38,6 +41,10 @@ let acmeInitializing = false;
 let acmeInitPending = [];
 
 const ensureAcme = async acmeOptions => {
+    if (!settings) {
+        settings = new SettingsHandler({ db: db.database });
+    }
+
     if (acmeInitialized) {
         return true;
     }
@@ -75,10 +82,11 @@ const getAcmeAccount = async (acmeOptions, certHandler) => {
 
     const entryKey = `acme:account:${acmeOptions.key}`;
 
-    const settingsValue = await db.database.collection('settings').findOne({ key: entryKey });
+    const settingsValue = await settings.get(entryKey);
+
     // there is already an existing acme account, no need to create a new one
-    if (settingsValue && settingsValue.value) {
-        return settingsValue.value;
+    if (settingsValue) {
+        return settingsValue;
     }
 
     // account not found, create a new one
@@ -96,17 +104,16 @@ const getAcmeAccount = async (acmeOptions, certHandler) => {
 
     const account = await acme.accounts.create(accountOptions);
 
-    const r = await db.database.collection('settings').insertOne({
-        key: entryKey,
-        value: {
+    await settings.set(
+        entryKey,
+        {
             key: accountKey,
             account
         },
-        enumerable: false,
-        created: new Date()
-    });
+        { enumerable: false }
+    );
 
-    log.info('ACME', 'ACME account provisioned for %s (%s)', acmeOptions.key, r.insertedId);
+    log.info('ACME', 'ACME account provisioned for %s', acmeOptions.key);
 
     return {
         key: accountKey,

lib/api/settings.js

@@ -0,0 +1,189 @@
+'use strict';
+
+const Joi = require('joi');
+const tools = require('../tools');
+const roles = require('../roles');
+const { sessSchema, sessIPSchema } = require('../schemas');
+const { SettingsHandler } = require('../settings-handler');
+const consts = require('../consts');
+
+// allow overriding the following consts using the key format `const:archive:time`
+const supportedDefaults = ['ARCHIVE_TIME'];
+
+module.exports = (db, server) => {
+    let settingsHandler = new SettingsHandler({ db: db.database });
+
+    server.get(
+        { name: 'settings', path: '/settings' },
+        tools.asyncifyJson(async (req, res, next) => {
+            res.charSet('utf-8');
+
+            const schema = Joi.object().keys({
+                prefix: Joi.string().empty('').max(128),
+                sess: sessSchema,
+                ip: sessIPSchema
+            });
+
+            const result = schema.validate(req.params, {
+                abortEarly: false,
+                convert: true,
+                allowUnknown: true
+            });
+
+            if (result.error) {
+                res.status(400);
+                res.json({
+                    error: result.error.message,
+                    code: 'InputValidationError',
+                    details: tools.validationErrors(result)
+                });
+                return next();
+            }
+
+            let permission = roles.can(req.role).readAny('settings');
+            // permissions check
+            req.validate(permission);
+
+            let defaults = {};
+            for (let defaultKey of supportedDefaults) {
+                let constKey = `const:${defaultKey.toLowerCase().replace(/_/g, ':')}`;
+                defaults[constKey] = consts[defaultKey];
+            }
+
+            let settings = await settingsHandler.list(result.value.prefix);
+
+            let response = {
+                success: true,
+                prefix: result.value.prefix,
+                settings: Object.assign(defaults, settings)
+            };
+
+            res.json(response);
+            return next();
+        })
+    );
+
+    server.put(
+        '/settings/:key',
+        tools.asyncifyJson(async (req, res, next) => {
+            res.charSet('utf-8');
+
+            const schema = Joi.object().keys({
+                key: Joi.string().empty('').max(128).required(),
+                value: Joi.any().allow('', 0, false).required(),
+                sess: sessSchema,
+                ip: sessIPSchema
+            });
+
+            const result = schema.validate(req.params, {
+                abortEarly: false,
+                convert: true
+            });
+
+            if (result.error) {
+                res.status(400);
+                res.json({
+                    error: result.error.message,
+                    code: 'InputValidationError',
+                    details: tools.validationErrors(result)
+                });
+                return next();
+            }
+
+            // permissions check
+            let permission = roles.can(req.role).createAny('settings');
+            req.validate(permission);
+
+            result.value = permission.filter(result.value);
+
+            let key = result.value.key;
+            let value = result.value.value;
+
+            let storedValue;
+            try {
+                storedValue = await settingsHandler.set(key, value);
+            } catch (err) {
+                res.status(500);
+                res.json({
+                    error: 'MongoDB Error: ' + err.message,
+                    code: 'InternalDatabaseError'
+                });
+                return next();
+            }
+
+            res.json({
+                success: !!storedValue,
+                key
+            });
+            return next();
+        })
+    );
+
+    server.get(
+        '/settings/:key',
+        tools.asyncifyJson(async (req, res, next) => {
+            res.charSet('utf-8');
+
+            const schema = Joi.object().keys({
+                key: Joi.string().empty('').max(128).required(),
+                sess: sessSchema,
+                ip: sessIPSchema
+            });
+
+            const result = schema.validate(req.params, {
+                abortEarly: false,
+                convert: true
+            });
+
+            if (result.error) {
+                res.status(400);
+                res.json({
+                    error: result.error.message,
+                    code: 'InputValidationError',
+                    details: tools.validationErrors(result)
+                });
+                return next();
+            }
+
+            // permissions check
+            let permission = roles.can(req.role).readAny('settings');
+            req.validate(permission);
+
+            let key = result.value.key;
+
+            let defaultValue;
+            if (/^const:/.test(key)) {
+                // get default
+                let constKey = key
+                    .replace(/^const:/, '')
+                    .replace(/:/g, '_')
+                    .toUpperCase();
+
+                if (supportedDefaults.includes(constKey) && consts.hasOwnProperty(constKey)) {
+                    defaultValue = consts[constKey];
+                }
+            }
+
+            let value;
+            try {
+                value = await settingsHandler.get(key, { default: defaultValue });
+            } catch (err) {
+                res.status(500);
+                res.json({
+                    error: 'MongoDB Error: ' + err.message,
+                    code: 'InternalDatabaseError'
+                });
+                return next();
+            }
+
+            res.json({
+                success: value !== undefined,
+                key,
+                value,
+                error: value === undefined ? 'Key was not found' : undefined
+            });
+
+            return next();
+        })
+    );
+};

lib/cert-handler.js

@@ -117,10 +117,17 @@ class CertHandler {
             { $set: { '_acme.lastRenewalCheck': new Date() } },
             {
                 upsert: false,
-                returnDocument: 'after'
+                returnDocument: 'after',
+                projection: { _id: true }
             }
         );
-        return (r && r.value) || false;
+
+        if (r && r.value) {
+            // use getRecord to decrypt secrets
+            return await this.getRecord({ _id: r.value._id }, true);
+        }
+
+        return false;
     }
 
     async update(options, updates, updateOptions) {

lib/message-handler.js

@@ -15,6 +15,7 @@ const openpgp = require('openpgp');
 const parseDate = require('../imap-core/lib/parse-date');
 const log = require('npmlog');
 const packageData = require('../package.json');
+const { SettingsHandler } = require('./settings-handler');
 
 // index only the following headers for SEARCH
 const INDEXED_HEADERS = ['to', 'cc', 'subject', 'from', 'sender', 'reply-to', 'message-id', 'thread-index', 'list-id', 'delivered-to'];
@@ -58,6 +59,8 @@ class MessageHandler {
             bucket: 'audit',
             loggelf: message => this.loggelf(message)
         });
+
+        this.settingsHandler = new SettingsHandler({ db: this.database });
     }
 
     async getMailboxAsync(options) {
@@ -672,9 +675,11 @@ class MessageHandler {
         }
 
         if (options.archive) {
+            let archiveTime = await this.settingsHandler.get('const:archive:time', { default: consts.ARCHIVE_TIME });
+
             messageData.archived = curtime;
             messageData.exp = true;
-            messageData.rdate = curtime.getTime() + consts.ARCHIVE_TIME;
+            messageData.rdate = curtime.getTime() + archiveTime;
 
             let r;
             try {

lib/settings-handler.js

@@ -0,0 +1,114 @@
+'use strict';
+
+const { encrypt, decrypt } = require('./encrypt');
+
+class SettingsHandler {
+    constructor(opts) {
+        opts = opts || {};
+        this.db = opts.db;
+    }
+
+    async set(key, value, options) {
+        options = options || {};
+
+        let encrypted = false;
+        if (options.secret && options.encrypt) {
+            value = await encrypt(JSON.stringify(value), options.secret);
+        } else {
+            value = JSON.stringify(value);
+        }
+
+        let $set = {
+            key,
+            value
+        };
+
+        if (encrypted) {
+            $set.encrypted = true;
+        } else {
+            $set.encrypted = false;
+        }
+
+        let $setOnInsert = {
+            created: new Date()
+        };
+
+        if (options && 'enumerable' in options) {
+            $set.enumerable = !!options.enumerable;
+        } else {
+            // default for new keys
+            $setOnInsert.enumerable = true;
+        }
+
+        let r = await this.db.collection('settings').findOneAndUpdate(
+            {
+                key
+            },
+            {
+                $set,
+                $setOnInsert
+            },
+            { upsert: true, returnDocument: 'after' }
+        );
+
+        return r.value && r.value.value;
+    }
+
+    async get(key, options) {
+        options = options || {};
+        let row = await this.db.collection('settings').findOne({
+            key
+        });
+
+        if (row && row.encrypted && typeof row.value === 'string') {
+            if (!options.secret) {
+                throw new Error('Secret not provided for encrypted value');
+            }
+            let value = await decrypt(row.value, options.secret);
+            return JSON.parse(value);
+        } else if (row && typeof row.value === 'string') {
+            return JSON.parse(row.value);
+        }
+
+        return row ? row.value : options.default;
+    }
+
+    async del(key) {
+        return await this.db.collection('settings').deleteOne({
+            key
+        });
+    }
+
+    async list(prefix, options) {
+        options = options || {};
+        let query = { enumerable: true };
+        if (prefix) {
+            query.key = {
+                $regex: '^' + prefix.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&'),
+                $options: 'i'
+            };
+        }
+        let list = await this.db.collection('settings').find(query).project({ key: true, value: true }).toArray();
+        let results = {};
+        for (let row of list) {
+            try {
+                if (row && row.encrypted && typeof row.value === 'string') {
+                    if (!options.secret) {
+                        throw new Error('Secret not provided for encrypted value');
+                    }
+                    let value = await decrypt(row.value, options.secret);
+                    row.value = JSON.parse(value);
+                } else if (row && typeof row.value === 'string') {
+                    row.value = JSON.parse(row.value);
+                }
+
+                results[row.key] = row.value;
+            } catch (err) {
+                // ignore?
+            }
+        }
+        return results;
+    }
+}
+
+module.exports.SettingsHandler = SettingsHandler;

lib/tasks/acme-update.js

@@ -8,8 +8,6 @@ let run = async (task, data, options) => {
 
     let certData;
     while ((certData = await certHandler.getNextRenewal())) {
-        console.log('RENEWING %s %s', certData._id, certData.servername);
-
         let cert = await acquireCert(certData.servername, config.acme, certData, certHandler);
         log.verbose('Tasks', 'task=acme-update id=%s servername=%s status=%s', task._id, certData.servername, cert && cert.status);
     }

tasks.js

@@ -146,7 +146,7 @@ module.exports.start = callback => {
         }
         let collection = collections[collectionpos++];
         db[collection.type || 'database'].createCollection(collection.collection, collection.options, err => {
-            if (err) {
+            if (err && err.codeName !== 'NamespaceExists') {
                 log.error('Setup', 'Failed creating collection %s %s. %s', collectionpos, JSON.stringify(collection.collection), err.message);
             }
 
@@ -154,6 +154,26 @@ module.exports.start = callback => {
         });
     };
 
+    let deleteindexes = setupIndexes.deleteindexes;
+    let deleteindexpos = 0;
+    let deleteIndexes = next => {
+        if (deleteindexpos >= deleteindexes.length) {
+            return next();
+        }
+        let index = deleteindexes[deleteindexpos++];
+        db[index.type || 'database'].collection(index.collection).dropIndex(index.index, (err, r) => {
+            if (r && r.ok) {
+                log.info('Setup', 'Deleted index %s from %s', index.index, index.collection);
+            }
+
+            if (err && err.codeName !== 'IndexNotFound') {
+                log.error('Setup', 'Failed to delete index %s %s. %s', deleteindexpos, JSON.stringify(index.collection + '.' + index.index), err.message);
+            }
+
+            deleteIndexes(next);
+        });
+    };
+
     let indexes = setupIndexes.indexes;
     let indexpos = 0;
     let ensureIndexes = next => {
@@ -167,21 +187,13 @@ module.exports.start = callback => {
                 log.error('Setup', 'Failed creating index %s %s. %s', indexpos, JSON.stringify(index.collection + '.' + index.index.name), err.message);
             } else if (r.numIndexesAfter !== r.numIndexesBefore) {
                 log.verbose('Setup', 'Created index %s %s', indexpos, JSON.stringify(index.collection + '.' + index.index.name));
-            } else {
-                log.verbose(
-                    'Setup',
-                    'Skipped index %s %s: %s',
-                    indexpos,
-                    JSON.stringify(index.collection + '.' + index.index.name),
-                    r.note || 'No index added'
-                );
             }
 
             ensureIndexes(next);
         });
     };
 
-    gcLock.acquireLock('db_indexes', 1 * 60 * 1000, (err, lock) => {
+    gcLock.acquireLock('db_indexes', 5 * 60 * 1000, (err, lock) => {
         if (err) {
             log.error('GC', 'Failed to acquire lock error=%s', err.message);
             return start();
@@ -190,17 +202,19 @@ module.exports.start = callback => {
         }
 
         ensureCollections(() => {
-            ensureIndexes(() => {
-                // Do not release the indexing lock immediatelly
-                setTimeout(() => {
-                    gcLock.releaseLock(lock, err => {
-                        if (err) {
-                            console.error(lock);
-                            log.error('GC', 'Failed to release lock error=%s', err.message);
-                        }
-                    });
-                }, 60 * 1000);
-                return start();
+            deleteIndexes(() => {
+                ensureIndexes(() => {
+                    // Do not release the indexing lock immediatelly
+                    setTimeout(() => {
+                        gcLock.releaseLock(lock, err => {
+                            if (err) {
+                                console.error(lock);
+                                log.error('GC', 'Failed to release lock error=%s', err.message);
+                            }
+                        });
+                    }, 60 * 1000);
+                    return start();
+                });
             });
         });
     });