From 14d23511231ed962d17872a1ccad0ff5e66c0154 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Mon, 18 Jul 2022 20:24:04 +0300 Subject: [PATCH] v1.37.0 --- .eslintrc | 2 +- docs/api/openapi.yml | 13 ++ indexes.yaml | 8 + lib/api/messages.js | 17 ++ lib/bimi-handler.js | 374 ++++++++++++++++++++++++++++++++++++ package.json | 15 +- setup/01_install_commits.sh | 2 +- setup/08_install_haraka.sh | 4 +- tasks.js | 8 +- 9 files changed, 428 insertions(+), 15 deletions(-) create mode 100644 lib/bimi-handler.js diff --git a/.eslintrc b/.eslintrc index cf1fc4a8..8a83c996 100644 --- a/.eslintrc +++ b/.eslintrc @@ -7,6 +7,6 @@ }, "extends": ["nodemailer", "prettier"], "parserOptions": { - "ecmaVersion": 2018 + "ecmaVersion": 2020 } } diff --git a/docs/api/openapi.yml b/docs/api/openapi.yml index 52f622ab..c83137db 100644 --- a/docs/api/openapi.yml +++ b/docs/api/openapi.yml @@ -4831,6 +4831,19 @@ components: description: Attachments for the message verificationResults: $ref: '#/components/schemas/VerificationResults' + bimi: + type: object + description: BIMI logo info. If logo validation failed in any way, then this property is not set + properties: + certified: + type: boolean + description: If true, then this logo is from a VMC file + url: + type: string + description: URL of the resource the logo was retrieved from + image: + type: string + description: Data URL for the SVG image contentType: $ref: '#/components/schemas/ContentType' metaData: diff --git a/indexes.yaml b/indexes.yaml index bfae5282..6cd348c6 100644 --- a/indexes.yaml +++ b/indexes.yaml @@ -698,6 +698,14 @@ indexes: key: metadata.subject: 1 + - collection: bimi + index: + name: by_type + unique: true + key: + type: 1 + url: 1 + - collection: webhooks index: name: by_type diff --git a/lib/api/messages.js b/lib/api/messages.js index 50441b09..b0cdd472 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -939,6 +939,23 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti } if (messageData.verificationResults) { + if (messageData.verificationResults.bimi) { + try { + let bimiData = await db.database.collection('bimi').findOne({ _id: messageData.verificationResults.bimi }); + if (bimiData?.content && !bimiData?.error) { + response.bimi = { + certified: bimiData.type === 'authority', + url: bimiData.url, + image: `data:image/svg+xml;base64,${bimiData.content.toString('base64')}` + }; + } + } catch (err) { + log.error('BIMI', 'message=%s error=%s', messageData._id, err.message); + } + + delete messageData.verificationResults.bimi; + } + response.verificationResults = messageData.verificationResults; } diff --git a/lib/bimi-handler.js b/lib/bimi-handler.js new file mode 100644 index 00000000..dd261c84 --- /dev/null +++ b/lib/bimi-handler.js @@ -0,0 +1,374 @@ +'use strict'; + +const packageData = require('../package.json'); +const https = require('https'); +const { validateSvg } = require('mailauth/lib/bimi/validate-svg'); +const { vmc } = require('@postalsys/vmc'); +const { formatDomain, getAlignment } = require('mailauth/lib/tools'); + +class BimiHandler { + static create(options = {}) { + return new BimiHandler(options); + } + + constructor(options) { + this.options = options || {}; + + this.database = options.database; + } + + async download(url) { + if (!url) { + return false; + } + + const parsedUrl = new URL(url); + + let protoHandler; + switch (parsedUrl.protocol) { + case 'https:': + protoHandler = https; + break; + case 'http:': { + let error = new Error(`Only HTTPS addresses are allowed`); + error.code = 'PROTO_NOT_HTTPS'; + throw error; + } + default: { + let error = new Error(`Unknown protocol ${parsedUrl.protocol}`); + error.code = 'UNKNOWN_PROTO'; + throw error; + } + } + + const options = { + protocol: parsedUrl.protocol, + host: parsedUrl.host, + headers: { + host: parsedUrl.host, + 'User-Agent': `${packageData.name}/${packageData.version} (+${packageData.homepage}` + }, + servername: parsedUrl.hostname, + port: 443, + path: parsedUrl.pathname, + method: 'GET', + rejectUnauthorized: true + }; + + return new Promise((resolve, reject) => { + const req = protoHandler.request(options, res => { + let chunks = [], + chunklen = 0; + res.on('readable', () => { + let chunk; + while ((chunk = res.read()) !== null) { + chunks.push(chunk); + chunklen += chunk.length; + } + }); + res.on('end', () => { + let data = Buffer.concat(chunks, chunklen); + if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300) { + let err = new Error(`Invalid response code ${res.statusCode || '-'}`); + + if (res.headers.location && res.statusCode >= 300 && res.statusCode < 400) { + err.code = 'REDIRECT_NOT_ALLOWED'; + err.details = { + code: res.statusCode, + location: res.headers.location + }; + } else { + err.code = 'HTTP_STATUS_' + (res.statusCode || 'NA'); + } + return reject(err); + } + resolve(data); + }); + res.on('error', err => reject(err)); + }); + + req.on('error', err => { + reject(err); + }); + req.end(); + }); + } + + async getBimiData(url, type) { + if (!url) { + return false; + } + + let bimiDocument = await this.database.collection('bimi').findOne({ url, type }); + + let bimiTtl = bimiDocument?.error ? 1 * 3600 * 1000 : 24 * 3600 * 1000; + + if (bimiDocument && bimiDocument?.updated > new Date(Date.now() - bimiTtl)) { + if (bimiDocument.error) { + let error = new Error(bimiDocument.error.message); + if (bimiDocument.error.details) { + error.details = bimiDocument.error.details; + } + if (bimiDocument.error.code) { + error.code = bimiDocument.error.code; + } + throw error; + } + + return bimiDocument; + } + + bimiDocument = { + url, + type, + updated: new Date() + }; + + // Step 1. Download + + let file; + try { + file = await this.download(url); + } catch (err) { + bimiDocument.error = { + message: err.message, + details: err.details, + code: err.code + }; + + try { + await this.database.collection('bimi').updateOne( + { + url, + type + }, + { + $set: bimiDocument, + $setOnInsert: { + created: new Date() + } + }, + { upsert: true } + ); + } catch (err) { + // ignore + console.error(3, err); + } + + throw err; + } + + // Step 2. Validate VMC + if (type === 'authority') { + try { + let vmcData = await vmc(file); + + if (!vmcData.logoFile) { + let error = new Error('VMC does not contain a logo file'); + error.code = 'MISSING_VMC_LOGO'; + throw error; + } + + if (vmcData?.mediaType?.toLowerCase() !== 'image/svg+xml') { + let error = new Error('Invalid media type for the logo file'); + error.details = { + mediaType: vmcData.mediaType + }; + error.code = 'INVALID_MEDIATYPE'; + throw error; + } + + if (!vmcData.validHash) { + let error = new Error('VMC hash does not match logo file'); + error.details = { + hashAlgo: vmcData.hashAlgo, + hashValue: vmcData.hashValue, + logoFile: vmcData.logoFile + }; + error.code = 'INVALID_LOGO_HASH'; + throw error; + } + + bimiDocument.content = Buffer.from(vmcData.logoFile, 'base64'); + bimiDocument.vmc = vmcData; + } catch (err) { + bimiDocument.error = { + message: err.message, + details: err.details, + code: err.code + }; + + try { + await this.database.collection('bimi').updateOne( + { + type, + url + }, + { + $set: bimiDocument, + $setOnInsert: { + created: new Date() + } + }, + { upsert: true } + ); + } catch (err) { + // ignore + console.error(1, err); + } + + throw err; + } + } else { + bimiDocument.content = file; + } + + // Step 3. Validate SVG + + try { + validateSvg(bimiDocument.content); + } catch (err) { + let error = new Error('VMC logo SVG validation failed'); + error.details = Object.assign( + { + message: err.message + }, + error.details || {}, + err.code ? { code: err.code } : {} + ); + error.code = 'SVG_VALIDATION_FAILED'; + + bimiDocument.error = { + message: error.message, + details: error.details, + code: error.code + }; + + try { + await this.database.collection('bimi').updateOne( + { + type, + url + }, + { + $set: bimiDocument, + $setOnInsert: { + created: new Date() + } + }, + { upsert: true } + ); + } catch (err) { + // ignore + console.error(2, err); + } + + throw error; + } + + // clear pending errors + bimiDocument.error = null; + + let r = await this.database.collection('bimi').findOneAndUpdate( + { + type, + url + }, + { + $set: bimiDocument, + $setOnInsert: { + created: new Date() + } + }, + { upsert: true, returnDocument: 'after' } + ); + + return r && r.value; + } + + async getInfo(bimiData) { + let [ + { reason: locationError, value: locationValue, status: locationStatus }, + { reason: authorityError, value: authorityValue, status: authorityStatus } + ] = await Promise.allSettled([this.getBimiData(bimiData.location, 'location'), this.getBimiData(bimiData.authority, 'authority')]); + + if (locationError) { + throw locationError; + } + + if (authorityError) { + throw locationError; + } + + if (authorityStatus === 'fulfilled' && authorityValue) { + let selector = bimiData.status?.header?.selector; + let d = bimiData.status?.header?.d; + + // validate domain + let selectorSet = []; + let domainSet = []; + authorityValue.vmc?.certificate?.subjectAltName?.map(formatDomain)?.forEach(domain => { + if (/\b_bimi\./.test(domain)) { + selectorSet.push(domain); + } else { + domainSet.push(domain); + } + }); + + let domainVerified = false; + + if (selector && selectorSet.includes(formatDomain(`${selector}._bimi.${d}`))) { + domainVerified = true; + } else { + let alignedDomain = getAlignment(d, domainSet, false); + if (alignedDomain) { + domainVerified = true; + } + } + + if (!domainVerified) { + let error = new Error('Domain can not be verified'); + error.details = { + subjectAltName: authorityValue.vmc?.certificate?.subjectAltName, + selector, + d + }; + error.code = 'VMC_DOMAIN_MISMATCH'; + throw error; + } + + return authorityValue; + } + + return locationStatus === 'fulfilled' && locationValue; + } +} + +module.exports = BimiHandler; + +/* +const db = require('./db'); + +db.connect(() => { + let bimi = BimiHandler.create({ + database: db.database + }); + + bimi.getInfo({ + status: { + header: { + selector: 'default', + d: 'zone.ee' + }, + result: 'pass' + }, + rr: 'v=BIMI1; l=https://zone.ee/common/img/zone_profile_square_bimi.svg;a=https://zone.ee/.well-known/bimi.pem', + location: 'https://zone.ee/common/img/zone_profile_square_bimi.svg', + authority: 'https://zone.ee/.well-known/bimi.pem', + info: 'bimi=pass header.selector=default header.d=zone.ee' + }) + .then(result => console.log(require('util').inspect(result, false, 22))) + .catch(err => console.error(err)) + .finally(() => process.exit()); +}); +*/ diff --git a/package.json b/package.json index 0e55414f..221e2e6c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "wildduck", - "version": "1.36.2", + "version": "1.37.0", "description": "IMAP/POP3 server built with Node.js and MongoDB", "main": "server.js", "scripts": { @@ -21,11 +21,12 @@ "email": "andris@kreata.ee" }, "license": "EUPL-1.2", + "homepage": "https://wildduck.email/", "devDependencies": { "ajv": "8.11.0", "chai": "4.3.6", "docsify-cli": "4.4.4", - "eslint": "8.19.0", + "eslint": "8.20.0", "eslint-config-nodemailer": "1.2.0", "eslint-config-prettier": "8.5.0", "grunt": "1.5.3", @@ -43,6 +44,7 @@ "dependencies": { "@fidm/x509": "1.2.1", "@phc/pbkdf2": "1.1.14", + "@postalsys/vmc": "1.0.4", "@root/acme": "3.1.0", "@root/csr": "0.8.1", "accesscontrol": "2.2.1", @@ -59,7 +61,7 @@ "humanname": "0.2.2", "iconv-lite": "0.6.3", "ioredfour": "1.2.0-ioredis-06", - "ioredis": "5.1.0", + "ioredis": "5.2.1", "ipaddr.js": "2.0.1", "isemail": "3.2.0", "joi": "17.6.0", @@ -68,10 +70,11 @@ "libbase64": "1.2.1", "libmime": "5.1.0", "libqp": "1.1.0", + "mailauth": "3.0.2", "mailsplit": "5.3.2", "mobileconfig": "2.4.0", "mongo-cursor-pagination": "7.6.1", - "mongodb": "4.7.0", + "mongodb": "4.8.0", "mongodb-extended-json": "1.11.1", "node-forge": "1.3.1", "node-html-parser": "5.3.3", @@ -81,7 +84,7 @@ "pem-jwk": "2.0.0", "punycode": "2.1.1", "pwnedpasswords": "1.0.6", - "qrcode": "1.5.0", + "qrcode": "1.5.1", "restify": "8.6.1", "restify-cors-middleware2": "2.1.2", "restify-logger": "2.0.1", @@ -101,6 +104,6 @@ "url": "git://github.com/wildduck-email/wildduck.git" }, "engines": { - "node": ">=12.0.0 <17" + "node": ">=16.0.0 <17" } } diff --git a/setup/01_install_commits.sh b/setup/01_install_commits.sh index 4dfdbf8c..b272291a 100755 --- a/setup/01_install_commits.sh +++ b/setup/01_install_commits.sh @@ -11,6 +11,6 @@ ZONEMTA_COMMIT="a08d064e6a50ea59ca4be3b8e541f2ba279a16a9" # zone-mta-template WEBMAIL_COMMIT="3371984a32a7942d7859c3fcde923cf62484e7fa" WILDDUCK_ZONEMTA_COMMIT="05cc573da50d63abff2fd0b17cab483b21729fb7" WILDDUCK_HARAKA_COMMIT="517a6d5d5e16e7a70f397a67ca99207434831a08" -HARAKA_VERSION="2.8.27" +HARAKA_VERSION="2.8.28" echo -e "\n-- Executing ${ORANGE}${OURNAME}${NC} subscript --" diff --git a/setup/08_install_haraka.sh b/setup/08_install_haraka.sh index d7428ac4..ada65f50 100755 --- a/setup/08_install_haraka.sh +++ b/setup/08_install_haraka.sh @@ -33,10 +33,10 @@ chmod +x "/var/opt/haraka-plugin-wildduck.git/hooks/update" echo "deploy ALL = (root) NOPASSWD: $SYSTEMCTL_PATH restart haraka" >> /etc/sudoers.d/wildduck cd -npm install --production --no-optional --no-package-lock --no-audit --ignore-scripts --no-shrinkwrap --unsafe-perm -g Haraka@$HARAKA_VERSION +npm install --production --no-optional --no-package-lock --no-audit --no-shrinkwrap --unsafe-perm -g Haraka@$HARAKA_VERSION haraka -i /opt/haraka cd /opt/haraka -npm install --production --no-optional --no-package-lock --no-audit --ignore-scripts --no-shrinkwrap --unsafe-perm --save haraka-plugin-rspamd haraka-plugin-redis Haraka@$HARAKA_VERSION +npm install --production --no-optional --no-package-lock --no-audit --no-shrinkwrap --unsafe-perm --save haraka-plugin-rspamd haraka-plugin-redis haraka-plugin-mailauth Haraka@$HARAKA_VERSION # Haraka WildDuck plugin. Install as separate repo as it can be edited more easily later mkdir -p plugins/wildduck diff --git a/tasks.js b/tasks.js index eaadd5ad..c9cd8ac1 100644 --- a/tasks.js +++ b/tasks.js @@ -168,7 +168,7 @@ module.exports.start = callback => { log.info('Setup', 'Deleted index %s from %s', index.index, index.collection); } - if (err && err.codeName !== 'IndexNotFound') { + if (err && err.codeName !== 'IndexNotFound' && err.codeName !== 'NamespaceNotFound') { log.error('Setup', 'Failed to delete index %s %s. %s', deleteindexpos, JSON.stringify(index.collection + '.' + index.index), err.message); } @@ -185,9 +185,9 @@ module.exports.start = callback => { } let index = indexes[indexpos++]; db[index.type || 'database'].collection(index.collection).createIndexes([index.index], (err, r) => { - if (err) { + if (err && err.codeName !== 'IndexOptionsConflict') { log.error('Setup', 'Failed creating index %s %s. %s', indexpos, JSON.stringify(index.collection + '.' + index.index.name), err.message); - } else if (r.numIndexesAfter !== r.numIndexesBefore) { + } else if (!err && r.numIndexesAfter !== r.numIndexesBefore) { log.verbose('Setup', 'Created index %s %s', indexpos, JSON.stringify(index.collection + '.' + index.index.name)); } @@ -210,7 +210,6 @@ module.exports.start = callback => { setTimeout(() => { gcLock.releaseLock(lock, err => { if (err) { - console.error(lock); log.error('GC', 'Failed to release lock error=%s', err.message); } }); @@ -247,7 +246,6 @@ function clearExpiredMessages() { let done = () => { gcLock.releaseLock(lock, err => { if (err) { - console.error(lock); log.error('GC', 'Failed to release lock error=%s', err.message); } gcTimeout = setTimeout(clearExpiredMessages, consts.GC_INTERVAL);