From 9b54fc4a03f157368410b57a49ac93da08ed6438 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Wed, 18 Oct 2017 16:32:01 +0300 Subject: [PATCH] Parse message just once for multiple deliveries --- examples/push-message.js | 119 ++++++++++++++++--------------- imap-core/lib/indexer/indexer.js | 7 +- lib/filter-handler.js | 118 +++++++++++++++++++++++++----- lib/message-handler.js | 10 ++- lmtp.js | 12 +++- 5 files changed, 185 insertions(+), 81 deletions(-) diff --git a/examples/push-message.js b/examples/push-message.js index 885eb916..2f84cb52 100644 --- a/examples/push-message.js +++ b/examples/push-message.js @@ -2,11 +2,11 @@ 'use strict'; -const recipient = process.argv[2]; -const total = Number(process.argv[3]) || 1; +const recipients = process.argv.slice(2); +const total = 1; -if (!recipient) { - console.error('Usage: node example.com username@exmaple.com'); // eslint-disable-line no-console +if (!recipients || !recipients.length) { + console.error('Usage: node example.com recipient1@exmaple.com [recipient2@exmaple.com...]'); // eslint-disable-line no-console return process.exit(1); } @@ -28,64 +28,71 @@ let sent = 0; let startTime = Date.now(); function send() { - - transporter.sendMail({ - envelope: { - from: 'andris@kreata.ee', - to: [recipient] - }, - - headers: { - // set to Yes to send this message to Junk folder - 'x-rspamd-spam': 'No' - }, - - from: 'Kärbes 🐧 ', - to: 'Ämblik 🦉 <' + recipient + '>, andmekala@hot.ee, Müriaad Polüteism ', - subject: 'Test ööö message [' + Date.now() + ']', - text: 'Hello world! Current time is ' + new Date().toString(), - html: '

Hello world! Current time is ' + new Date().toString() + '

', - attachments: [ - - // attachment as plaintext - { - filename: 'notes.txt', - content: 'Some notes about this e-mail', - contentType: 'text/plain' // optional, would be detected from the filename + transporter.sendMail( + { + envelope: { + from: 'andris@kreata.ee', + to: recipients }, - // Small Binary Buffer attachment, should be kept with message - { - filename: 'image.png', - content: new Buffer('iVBORw0KGgoAAAANSUhEUgAAABAAAAAQAQMAAAAlPW0iAAAABlBMVEUAAAD/' + - '//+l2Z/dAAAAM0lEQVR4nGP4/5/h/1+G/58ZDrAz3D/McH8yw83NDDeNGe4U' + - 'g9C9zwz3gVLMDA/A6P9/AFGGFyjOXZtQAAAAAElFTkSuQmCC', 'base64'), - - cid: 'note@example.com' // should be as unique as possible + headers: { + // set to Yes to send this message to Junk folder + 'x-rspamd-spam': 'No' }, - // Large Binary Buffer attachment, should be kept separately - { - path: __dirname + '/swan.jpg', - filename: 'swän.jpg' + from: 'Kärbes 🐧 ', + to: recipients.map((rcpt, i) => ({ name: 'Recipient #' + (i + 1), address: rcpt })), + subject: 'Test ööö message [' + Date.now() + ']', + text: 'Hello world! Current time is ' + new Date().toString(), + html: + '

Hello world! Current time is ' + + new Date().toString() + + '

', + attachments: [ + // attachment as plaintext + { + filename: 'notes.txt', + content: 'Some notes about this e-mail', + contentType: 'text/plain' // optional, would be detected from the filename + }, + + // Small Binary Buffer attachment, should be kept with message + { + filename: 'image.png', + content: new Buffer( + 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQAQMAAAAlPW0iAAAABlBMVEUAAAD/' + + '//+l2Z/dAAAAM0lEQVR4nGP4/5/h/1+G/58ZDrAz3D/McH8yw83NDDeNGe4U' + + 'g9C9zwz3gVLMDA/A6P9/AFGGFyjOXZtQAAAAAElFTkSuQmCC', + 'base64' + ), + + cid: 'note@example.com' // should be as unique as possible + }, + + // Large Binary Buffer attachment, should be kept separately + { + path: __dirname + '/swan.jpg', + filename: 'swän.jpg' + } + ] + }, + (err, info) => { + if (err && err.response) { + console.log('Message failed: %s', err.response); + } else if (err) { + console.log(err); + } else { + console.log(info); + } + sent++; + if (sent >= total) { + console.log('Sent %s messages in %s s', sent, (Date.now() - startTime) / 1000); + return transporter.close(); + } else { + send(); } - ] - }, (err, info) => { - if (err && err.response) { - console.log('Message failed: %s', err.response); - } else if (err) { - console.log(err); - } else { - console.log(info); } - sent++; - if (sent >= total) { - console.log('Sent %s messages in %s s', sent, (Date.now() - startTime) / 1000); - return transporter.close(); - } else { - send(); - } - }); + ); } send(); /* diff --git a/imap-core/lib/indexer/indexer.js b/imap-core/lib/indexer/indexer.js index f8eab794..c8561944 100644 --- a/imap-core/lib/indexer/indexer.js +++ b/imap-core/lib/indexer/indexer.js @@ -269,7 +269,7 @@ class Indexer { /** * Decode text/plain and text/html parts, separate node bodies from the tree */ - getMaildata(messageId, mimeTree) { + getMaildata(mimeTree) { let magic = parseInt(crypto.randomBytes(2).toString('hex'), 16); let maildata = { nodes: [], @@ -446,7 +446,7 @@ class Indexer { str.replace(/\bcid:([^\s"']+)/g, (match, cid) => { if (cidMap.has(cid)) { let attachment = cidMap.get(cid); - return 'attachment:' + messageId + '/' + attachment.id.toString(); + return 'attachment:' + attachment.id.toString(); } return match; }); @@ -464,9 +464,10 @@ class Indexer { /** * Stores attachments to GridStore */ - storeNodeBodies(messageId, maildata, mimeTree, callback) { + storeNodeBodies(maildata, mimeTree, callback) { let pos = 0; let nodes = maildata.nodes; + mimeTree.attachmentMap = {}; let storeNode = () => { if (pos >= nodes.length) { diff --git a/lib/filter-handler.js b/lib/filter-handler.js index 967810c0..69d2dd1b 100644 --- a/lib/filter-handler.js +++ b/lib/filter-handler.js @@ -31,8 +31,9 @@ class FilterHandler { this.users = options.users || options.database; this.redis = options.redis; this.messageHandler = options.messageHandler; - this.spamChecks = options.spamChecks; - this.spamHeaderKeys = options.spamHeaderKeys; + + this.spamChecks = options.spamChecks || prepareSpamChecks(defaultSpamHeaderKeys); + this.spamHeaderKeys = options.spamHeaderKeys || this.spamChecks.map(check => check.key); } getUserData(address, callback) { @@ -121,16 +122,45 @@ class FilterHandler { let chunks = options.chunks; let chunklen = options.chunklen; - let raw = Buffer.concat([extraHeader].concat(chunks), chunklen + extraHeader.length); - let prepared = this.messageHandler.prepareMessage({ - raw, - indexedHeaders: this.spamHeaderKeys - }); + if (!chunks && options.raw) { + chunks = [options.raw]; + chunklen = options.raw.length; + } - console.log(require('util').inspect(prepared, false, 22)); + let prepared; - let maildata = this.messageHandler.indexer.getMaildata(prepared.id, prepared.mimeTree); + if (options.mimeTree) { + if (options.mimeTree && options.mimeTree.header) { + // remove old headers + if (/^Delivered-To/.test(options.mimeTree.header[0])) { + options.mimeTree.header.shift(); + } + if (/^Return-Path/.test(options.mimeTree.header[0])) { + options.mimeTree.header.shift(); + } + } + prepared = this.messageHandler.prepareMessage({ + mimeTree: options.mimeTree, + indexedHeaders: this.spamHeaderKeys + }); + } else { + let raw = Buffer.concat(chunks, chunklen); + prepared = this.messageHandler.prepareMessage({ + raw, + indexedHeaders: this.spamHeaderKeys + }); + } + + prepared.mimeTree.header.unshift('Return-Path: <' + sender + '>'); + prepared.mimeTree.header.unshift('Delivered-To: ' + recipient); + + prepared.mimeTree.parsedHeader['return-path'] = '<' + sender + '>'; + prepared.mimeTree.parsedHeader['delivered-to'] = '<' + recipient + '>'; + + prepared.size = this.messageHandler.indexer.getSize(prepared.mimeTree); + + let maildata = options.maildata || this.messageHandler.indexer.getMaildata(prepared.mimeTree); // default flags are empty let flags = []; @@ -358,22 +388,31 @@ class FilterHandler { skipExisting: true }; - this.messageHandler.encryptMessage(userData.encryptMessages ? userData.pubKey : false, raw, (err, encrypted) => { + this.messageHandler.encryptMessage(userData.encryptMessages ? userData.pubKey : false, { chunks, chunklen }, (err, encrypted) => { if (!err && encrypted) { messageOpts.prepared = this.messageHandler.prepareMessage({ - raw: encrypted, + raw: Buffer.concat([extraHeader, encrypted]), indexedHeaders: this.spamHeaderKeys }); - messageOpts.maildata = this.messageHandler.indexer.getMaildata(messageOpts.prepared.id, messageOpts.prepared.mimeTree); + messageOpts.maildata = this.messageHandler.indexer.getMaildata(messageOpts.prepared.mimeTree); } - this.messageHandler.add(messageOpts, (err, inserted, info) => + this.messageHandler.add(messageOpts, (err, inserted, info) => { // push to response list - callback(null, { - userData, - response: err ? err : 'Message stored as ' + info.id.toString() - }) - ); + callback( + null, + { + userData, + response: err ? err : 'Message stored as ' + info.id.toString() + }, + !encrypted + ? { + mimeTree: messageOpts.prepared.mimeTree, + maildata: messageOpts.maildata + } + : false + ); + }); }); }); }); @@ -462,3 +501,46 @@ function checkFilter(filter, prepared, maildata) { } module.exports = FilterHandler; + +function prepareSpamChecks(spamHeader) { + return (Array.isArray(spamHeader) ? spamHeader : [].concat(spamHeader || [])) + .map(header => { + if (!header) { + return false; + } + + // If only a single header key is specified, check if it matches Yes + if (typeof header === 'string') { + header = { + key: header, + value: '^yes', + target: '\\Junk' + }; + } + + let key = (header.key || '') + .toString() + .trim() + .toLowerCase(); + let value = (header.value || '').toString().trim(); + try { + if (value) { + value = new RegExp(value, 'i'); + value.isRegex = true; + } + } catch (E) { + value = false; + log.error('LMTP', 'Failed loading spam header rule %s. %s', JSON.stringify(header.value), E.message); + } + if (!key || !value) { + return false; + } + let target = (header.target || '').toString().trim() || 'INBOX'; + return { + key, + value, + target + }; + }) + .filter(check => check); +} diff --git a/lib/message-handler.js b/lib/message-handler.js index fdba6dd1..32e4b513 100644 --- a/lib/message-handler.js +++ b/lib/message-handler.js @@ -95,7 +95,7 @@ class MessageHandler { let headers = prepared.headers; let flags = Array.isArray(options.flags) ? options.flags : [].concat(options.flags || []); - let maildata = options.maildata || this.indexer.getMaildata(id, mimeTree); + let maildata = options.maildata || this.indexer.getMaildata(mimeTree); this.getMailbox(options, (err, mailboxData) => { if (err) { @@ -129,7 +129,7 @@ class MessageHandler { this.attachmentStorage.deleteMany(attachmentIds, maildata.magic, () => callback(...args)); }; - this.indexer.storeNodeBodies(id, maildata, mimeTree, err => { + this.indexer.storeNodeBodies(maildata, mimeTree, err => { if (err) { return cleanup(err); } @@ -879,7 +879,7 @@ class MessageHandler { prepareMessage(options) { let id = new ObjectID(); - let mimeTree = this.indexer.parseMimeTree(options.raw); + let mimeTree = options.mimeTree || this.indexer.parseMimeTree(options.raw); let size = this.indexer.getSize(mimeTree); let bodystructure = this.indexer.getBodyStructure(mimeTree); @@ -1194,6 +1194,10 @@ class MessageHandler { return callback(null, false); } + if (raw && Array.isArray(raw.chunks) && raw.chunklen) { + raw = Buffer.concat(raw.chunks, raw.chunklen); + } + let lastBytes = []; let headerEnd = raw.length; let headerLength = 0; diff --git a/lmtp.js b/lmtp.js index c3b85dee..79fadea9 100644 --- a/lmtp.js +++ b/lmtp.js @@ -140,6 +140,8 @@ const serverOptions = { let transactionId = new ObjectID(); + let prepared = false; + let storeNext = () => { if (stored >= users.length) { return callback(null, responses.map(r => r.response)); @@ -157,6 +159,8 @@ const serverOptions = { filterHandler.process( { + mimeTree: prepared && prepared.mimeTree, + maildata: prepared && prepared.maildata, user: userData, sender, recipient, @@ -174,13 +178,19 @@ const serverOptions = { time: Date.now() } }, - (err, response) => { + (err, response, preparedResponse) => { if (err) { // ??? } + if (response) { responses.push(response); } + + if (!prepared && preparedResponse) { + prepared = preparedResponse; + } + setImmediate(storeNext); } );