v1.37.0

.eslintrc

@@ -7,6 +7,6 @@
     },
     "extends": ["nodemailer", "prettier"],
     "parserOptions": {
-        "ecmaVersion": 2018
+        "ecmaVersion": 2020
     }
 }

docs/api/openapi.yml

@@ -4831,6 +4831,19 @@ components:
                     description: Attachments for the message
                 verificationResults:
                     $ref: '#/components/schemas/VerificationResults'
+                bimi:
+                    type: object
+                    description: BIMI logo info. If logo validation failed in any way, then this property is not set
+                    properties:
+                        certified:
+                            type: boolean
+                            description: If true, then this logo is from a VMC file
+                        url:
+                            type: string
+                            description: URL of the resource the logo was retrieved from
+                        image:
+                            type: string
+                            description: Data URL for the SVG image
                 contentType:
                     $ref: '#/components/schemas/ContentType'
                 metaData:

indexes.yaml

@@ -698,6 +698,14 @@ indexes:
           key:
               metadata.subject: 1
 
+    - collection: bimi
+      index:
+          name: by_type
+          unique: true
+          key:
+              type: 1
+              url: 1
+
     - collection: webhooks
       index:
           name: by_type

lib/api/messages.js

@@ -939,6 +939,23 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti
             }
 
             if (messageData.verificationResults) {
+                if (messageData.verificationResults.bimi) {
+                    try {
+                        let bimiData = await db.database.collection('bimi').findOne({ _id: messageData.verificationResults.bimi });
+                        if (bimiData?.content && !bimiData?.error) {
+                            response.bimi = {
+                                certified: bimiData.type === 'authority',
+                                url: bimiData.url,
+                                image: `data:image/svg+xml;base64,${bimiData.content.toString('base64')}`
+                            };
+                        }
+                    } catch (err) {
+                        log.error('BIMI', 'message=%s error=%s', messageData._id, err.message);
+                    }
+
+                    delete messageData.verificationResults.bimi;
+                }
+
                 response.verificationResults = messageData.verificationResults;
             }
 

lib/bimi-handler.js

@@ -0,0 +1,374 @@
+'use strict';
+
+const packageData = require('../package.json');
+const https = require('https');
+const { validateSvg } = require('mailauth/lib/bimi/validate-svg');
+const { vmc } = require('@postalsys/vmc');
+const { formatDomain, getAlignment } = require('mailauth/lib/tools');
+
+class BimiHandler {
+    static create(options = {}) {
+        return new BimiHandler(options);
+    }
+
+    constructor(options) {
+        this.options = options || {};
+
+        this.database = options.database;
+    }
+
+    async download(url) {
+        if (!url) {
+            return false;
+        }
+
+        const parsedUrl = new URL(url);
+
+        let protoHandler;
+        switch (parsedUrl.protocol) {
+            case 'https:':
+                protoHandler = https;
+                break;
+            case 'http:': {
+                let error = new Error(`Only HTTPS addresses are allowed`);
+                error.code = 'PROTO_NOT_HTTPS';
+                throw error;
+            }
+            default: {
+                let error = new Error(`Unknown protocol ${parsedUrl.protocol}`);
+                error.code = 'UNKNOWN_PROTO';
+                throw error;
+            }
+        }
+
+        const options = {
+            protocol: parsedUrl.protocol,
+            host: parsedUrl.host,
+            headers: {
+                host: parsedUrl.host,
+                'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage}`
+            },
+            servername: parsedUrl.hostname,
+            port: 443,
+            path: parsedUrl.pathname,
+            method: 'GET',
+            rejectUnauthorized: true
+        };
+
+        return new Promise((resolve, reject) => {
+            const req = protoHandler.request(options, res => {
+                let chunks = [],
+                    chunklen = 0;
+                res.on('readable', () => {
+                    let chunk;
+                    while ((chunk = res.read()) !== null) {
+                        chunks.push(chunk);
+                        chunklen += chunk.length;
+                    }
+                });
+                res.on('end', () => {
+                    let data = Buffer.concat(chunks, chunklen);
+                    if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300) {
+                        let err = new Error(`Invalid response code ${res.statusCode || '-'}`);
+
+                        if (res.headers.location && res.statusCode >= 300 && res.statusCode < 400) {
+                            err.code = 'REDIRECT_NOT_ALLOWED';
+                            err.details = {
+                                code: res.statusCode,
+                                location: res.headers.location
+                            };
+                        } else {
+                            err.code = 'HTTP_STATUS_' + (res.statusCode || 'NA');
+                        }
+                        return reject(err);
+                    }
+                    resolve(data);
+                });
+                res.on('error', err => reject(err));
+            });
+
+            req.on('error', err => {
+                reject(err);
+            });
+            req.end();
+        });
+    }
+
+    async getBimiData(url, type) {
+        if (!url) {
+            return false;
+        }
+
+        let bimiDocument = await this.database.collection('bimi').findOne({ url, type });
+
+        let bimiTtl = bimiDocument?.error ? 1 * 3600 * 1000 : 24 * 3600 * 1000;
+
+        if (bimiDocument && bimiDocument?.updated > new Date(Date.now() - bimiTtl)) {
+            if (bimiDocument.error) {
+                let error = new Error(bimiDocument.error.message);
+                if (bimiDocument.error.details) {
+                    error.details = bimiDocument.error.details;
+                }
+                if (bimiDocument.error.code) {
+                    error.code = bimiDocument.error.code;
+                }
+                throw error;
+            }
+
+            return bimiDocument;
+        }
+
+        bimiDocument = {
+            url,
+            type,
+            updated: new Date()
+        };
+
+        // Step 1. Download
+
+        let file;
+        try {
+            file = await this.download(url);
+        } catch (err) {
+            bimiDocument.error = {
+                message: err.message,
+                details: err.details,
+                code: err.code
+            };
+
+            try {
+                await this.database.collection('bimi').updateOne(
+                    {
+                        url,
+                        type
+                    },
+                    {
+                        $set: bimiDocument,
+                        $setOnInsert: {
+                            created: new Date()
+                        }
+                    },
+                    { upsert: true }
+                );
+            } catch (err) {
+                // ignore
+                console.error(3, err);
+            }
+
+            throw err;
+        }
+
+        // Step 2. Validate VMC
+        if (type === 'authority') {
+            try {
+                let vmcData = await vmc(file);
+
+                if (!vmcData.logoFile) {
+                    let error = new Error('VMC does not contain a logo file');
+                    error.code = 'MISSING_VMC_LOGO';
+                    throw error;
+                }
+
+                if (vmcData?.mediaType?.toLowerCase() !== 'image/svg+xml') {
+                    let error = new Error('Invalid media type for the logo file');
+                    error.details = {
+                        mediaType: vmcData.mediaType
+                    };
+                    error.code = 'INVALID_MEDIATYPE';
+                    throw error;
+                }
+
+                if (!vmcData.validHash) {
+                    let error = new Error('VMC hash does not match logo file');
+                    error.details = {
+                        hashAlgo: vmcData.hashAlgo,
+                        hashValue: vmcData.hashValue,
+                        logoFile: vmcData.logoFile
+                    };
+                    error.code = 'INVALID_LOGO_HASH';
+                    throw error;
+                }
+
+                bimiDocument.content = Buffer.from(vmcData.logoFile, 'base64');
+                bimiDocument.vmc = vmcData;
+            } catch (err) {
+                bimiDocument.error = {
+                    message: err.message,
+                    details: err.details,
+                    code: err.code
+                };
+
+                try {
+                    await this.database.collection('bimi').updateOne(
+                        {
+                            type,
+                            url
+                        },
+                        {
+                            $set: bimiDocument,
+                            $setOnInsert: {
+                                created: new Date()
+                            }
+                        },
+                        { upsert: true }
+                    );
+                } catch (err) {
+                    // ignore
+                    console.error(1, err);
+                }
+
+                throw err;
+            }
+        } else {
+            bimiDocument.content = file;
+        }
+
+        // Step 3. Validate SVG
+
+        try {
+            validateSvg(bimiDocument.content);
+        } catch (err) {
+            let error = new Error('VMC logo SVG validation failed');
+            error.details = Object.assign(
+                {
+                    message: err.message
+                },
+                error.details || {},
+                err.code ? { code: err.code } : {}
+            );
+            error.code = 'SVG_VALIDATION_FAILED';
+
+            bimiDocument.error = {
+                message: error.message,
+                details: error.details,
+                code: error.code
+            };
+
+            try {
+                await this.database.collection('bimi').updateOne(
+                    {
+                        type,
+                        url
+                    },
+                    {
+                        $set: bimiDocument,
+                        $setOnInsert: {
+                            created: new Date()
+                        }
+                    },
+                    { upsert: true }
+                );
+            } catch (err) {
+                // ignore
+                console.error(2, err);
+            }
+
+            throw error;
+        }
+
+        // clear pending errors
+        bimiDocument.error = null;
+
+        let r = await this.database.collection('bimi').findOneAndUpdate(
+            {
+                type,
+                url
+            },
+            {
+                $set: bimiDocument,
+                $setOnInsert: {
+                    created: new Date()
+                }
+            },
+            { upsert: true, returnDocument: 'after' }
+        );
+
+        return r && r.value;
+    }
+
+    async getInfo(bimiData) {
+        let [
+            { reason: locationError, value: locationValue, status: locationStatus },
+            { reason: authorityError, value: authorityValue, status: authorityStatus }
+        ] = await Promise.allSettled([this.getBimiData(bimiData.location, 'location'), this.getBimiData(bimiData.authority, 'authority')]);
+
+        if (locationError) {
+            throw locationError;
+        }
+
+        if (authorityError) {
+            throw locationError;
+        }
+
+        if (authorityStatus === 'fulfilled' && authorityValue) {
+            let selector = bimiData.status?.header?.selector;
+            let d = bimiData.status?.header?.d;
+
+            // validate domain
+            let selectorSet = [];
+            let domainSet = [];
+            authorityValue.vmc?.certificate?.subjectAltName?.map(formatDomain)?.forEach(domain => {
+                if (/\b_bimi\./.test(domain)) {
+                    selectorSet.push(domain);
+                } else {
+                    domainSet.push(domain);
+                }
+            });
+
+            let domainVerified = false;
+
+            if (selector && selectorSet.includes(formatDomain(`${selector}._bimi.${d}`))) {
+                domainVerified = true;
+            } else {
+                let alignedDomain = getAlignment(d, domainSet, false);
+                if (alignedDomain) {
+                    domainVerified = true;
+                }
+            }
+
+            if (!domainVerified) {
+                let error = new Error('Domain can not be verified');
+                error.details = {
+                    subjectAltName: authorityValue.vmc?.certificate?.subjectAltName,
+                    selector,
+                    d
+                };
+                error.code = 'VMC_DOMAIN_MISMATCH';
+                throw error;
+            }
+
+            return authorityValue;
+        }
+
+        return locationStatus === 'fulfilled' && locationValue;
+    }
+}
+
+module.exports = BimiHandler;
+
+/*
+const db = require('./db');
+
+db.connect(() => {
+    let bimi = BimiHandler.create({
+        database: db.database
+    });
+
+    bimi.getInfo({
+        status: {
+            header: {
+                selector: 'default',
+                d: 'zone.ee'
+            },
+            result: 'pass'
+        },
+        rr: 'v=BIMI1; l=https://zone.ee/common/img/zone_profile_square_bimi.svg;a=https://zone.ee/.well-known/bimi.pem',
+        location: 'https://zone.ee/common/img/zone_profile_square_bimi.svg',
+        authority: 'https://zone.ee/.well-known/bimi.pem',
+        info: 'bimi=pass header.selector=default header.d=zone.ee'
+    })
+        .then(result => console.log(require('util').inspect(result, false, 22)))
+        .catch(err => console.error(err))
+        .finally(() => process.exit());
+});
+*/

package.json

@@ -1,6 +1,6 @@
 {
     "name": "wildduck",
-    "version": "1.36.2",
+    "version": "1.37.0",
     "description": "IMAP/POP3 server built with Node.js and MongoDB",
     "main": "server.js",
     "scripts": {
@@ -21,11 +21,12 @@
         "email": "andris@kreata.ee"
     },
     "license": "EUPL-1.2",
+    "homepage": "https://wildduck.email/",
     "devDependencies": {
         "ajv": "8.11.0",
         "chai": "4.3.6",
         "docsify-cli": "4.4.4",
-        "eslint": "8.19.0",
+        "eslint": "8.20.0",
         "eslint-config-nodemailer": "1.2.0",
         "eslint-config-prettier": "8.5.0",
         "grunt": "1.5.3",
@@ -43,6 +44,7 @@
     "dependencies": {
         "@fidm/x509": "1.2.1",
         "@phc/pbkdf2": "1.1.14",
+        "@postalsys/vmc": "1.0.4",
         "@root/acme": "3.1.0",
         "@root/csr": "0.8.1",
         "accesscontrol": "2.2.1",
@@ -59,7 +61,7 @@
         "humanname": "0.2.2",
         "iconv-lite": "0.6.3",
         "ioredfour": "1.2.0-ioredis-06",
-        "ioredis": "5.1.0",
+        "ioredis": "5.2.1",
         "ipaddr.js": "2.0.1",
         "isemail": "3.2.0",
         "joi": "17.6.0",
@@ -68,10 +70,11 @@
         "libbase64": "1.2.1",
         "libmime": "5.1.0",
         "libqp": "1.1.0",
+        "mailauth": "3.0.2",
         "mailsplit": "5.3.2",
         "mobileconfig": "2.4.0",
         "mongo-cursor-pagination": "7.6.1",
-        "mongodb": "4.7.0",
+        "mongodb": "4.8.0",
         "mongodb-extended-json": "1.11.1",
         "node-forge": "1.3.1",
         "node-html-parser": "5.3.3",
@@ -81,7 +84,7 @@
         "pem-jwk": "2.0.0",
         "punycode": "2.1.1",
         "pwnedpasswords": "1.0.6",
-        "qrcode": "1.5.0",
+        "qrcode": "1.5.1",
         "restify": "8.6.1",
         "restify-cors-middleware2": "2.1.2",
         "restify-logger": "2.0.1",
@@ -101,6 +104,6 @@
         "url": "git://github.com/wildduck-email/wildduck.git"
     },
     "engines": {
-        "node": ">=12.0.0 <17"
+        "node": ">=16.0.0 <17"
     }
 }

setup/01_install_commits.sh

@@ -11,6 +11,6 @@ ZONEMTA_COMMIT="a08d064e6a50ea59ca4be3b8e541f2ba279a16a9" # zone-mta-template
 WEBMAIL_COMMIT="3371984a32a7942d7859c3fcde923cf62484e7fa"
 WILDDUCK_ZONEMTA_COMMIT="05cc573da50d63abff2fd0b17cab483b21729fb7"
 WILDDUCK_HARAKA_COMMIT="517a6d5d5e16e7a70f397a67ca99207434831a08"
-HARAKA_VERSION="2.8.27"
+HARAKA_VERSION="2.8.28"
 
 echo -e "\n-- Executing ${ORANGE}${OURNAME}${NC} subscript --"

setup/08_install_haraka.sh

@@ -33,10 +33,10 @@ chmod +x "/var/opt/haraka-plugin-wildduck.git/hooks/update"
 echo "deploy ALL = (root) NOPASSWD: $SYSTEMCTL_PATH restart haraka" >> /etc/sudoers.d/wildduck
 
 cd
-npm install --production --no-optional --no-package-lock --no-audit --ignore-scripts --no-shrinkwrap --unsafe-perm -g Haraka@$HARAKA_VERSION
+npm install --production --no-optional --no-package-lock --no-audit --no-shrinkwrap --unsafe-perm -g Haraka@$HARAKA_VERSION
 haraka -i /opt/haraka
 cd /opt/haraka
-npm install --production --no-optional --no-package-lock --no-audit --ignore-scripts --no-shrinkwrap --unsafe-perm --save haraka-plugin-rspamd haraka-plugin-redis Haraka@$HARAKA_VERSION
+npm install --production --no-optional --no-package-lock --no-audit --no-shrinkwrap --unsafe-perm --save haraka-plugin-rspamd haraka-plugin-redis haraka-plugin-mailauth Haraka@$HARAKA_VERSION
 
 # Haraka WildDuck plugin. Install as separate repo as it can be edited more easily later
 mkdir -p plugins/wildduck

tasks.js

@@ -168,7 +168,7 @@ module.exports.start = callback => {
                 log.info('Setup', 'Deleted index %s from %s', index.index, index.collection);
             }
 
-            if (err && err.codeName !== 'IndexNotFound') {
+            if (err && err.codeName !== 'IndexNotFound' && err.codeName !== 'NamespaceNotFound') {
                 log.error('Setup', 'Failed to delete index %s %s. %s', deleteindexpos, JSON.stringify(index.collection + '.' + index.index), err.message);
             }
 
@@ -185,9 +185,9 @@ module.exports.start = callback => {
         }
         let index = indexes[indexpos++];
         db[index.type || 'database'].collection(index.collection).createIndexes([index.index], (err, r) => {
-            if (err) {
+            if (err && err.codeName !== 'IndexOptionsConflict') {
                 log.error('Setup', 'Failed creating index %s %s. %s', indexpos, JSON.stringify(index.collection + '.' + index.index.name), err.message);
-            } else if (r.numIndexesAfter !== r.numIndexesBefore) {
+            } else if (!err && r.numIndexesAfter !== r.numIndexesBefore) {
                 log.verbose('Setup', 'Created index %s %s', indexpos, JSON.stringify(index.collection + '.' + index.index.name));
             }
 
@@ -210,7 +210,6 @@ module.exports.start = callback => {
                     setTimeout(() => {
                         gcLock.releaseLock(lock, err => {
                             if (err) {
-                                console.error(lock);
                                 log.error('GC', 'Failed to release lock error=%s', err.message);
                             }
                         });
@@ -247,7 +246,6 @@ function clearExpiredMessages() {
         let done = () => {
             gcLock.releaseLock(lock, err => {
                 if (err) {
-                    console.error(lock);
                     log.error('GC', 'Failed to release lock error=%s', err.message);
                 }
                 gcTimeout = setTimeout(clearExpiredMessages, consts.GC_INTERVAL);