nodemailer/wildduck
1
1
Fork 0
mirror of https://github.com/nodemailer/wildduck.git synced 2025-09-05 20:54:36 +08:00
Code Issues Projects Releases Packages Wiki Activity

Preprocess message in LMTP

Browse source
This commit is contained in:
Andris Reinman 2017-04-13 11:35:39 +03:00
parent f78562b184
commit 6406f479e6
13 changed files with 200 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
README.md
View file

@ -78,7 +78,7 @@ If a messages is deleted by a client this message gets marked as Seen and moved
### Does it work?
Yes, it does. You can run the server and get working IMAP and POP3 servers for mail store, SMTP server for pushing messages to the mail store and HTTP API server to create new users. All handled by Node.js, MongoDB and Redis, no additional dependencies needed. The IMAP server hosting уайлддак.орг uses a MongoDB replica set of 3 hosts.
Yes, it does. You can run the server and get working IMAP and POP3 servers for mail store, LMTP server for pushing messages to the mail store and HTTP API server to create new users. All handled by Node.js, MongoDB and Redis, no additional dependencies needed. Provided services can be disabled and enabled one by one so, for example you could process just IMAP in one host and LMTP in another.
### What are the killer features?
@ -230,7 +230,7 @@ The response for successful operation should look like this:
}
```
After you have registered a new address then SMTP maildrop server starts accepting mail for it and store the messages to the users mailbox.
After you have registered a new address then LMTP maildrop server starts accepting mail for it and store the messages to the users mailbox.
### POST /user/quota
@ -660,12 +660,16 @@ Create an email account and use your IMAP client to connect to it. To send mail
node examples/push-mail.js username@example.com
```
This should "deliver" a new message to the INBOX of _username@example.com_ by using the built-in SMTP maildrop interface. If your email client is connected then you should promptly see the new message.
This should "deliver" a new message to the INBOX of _username@example.com_ by using the built-in LMTP maildrop interface. If your email client is connected then you should promptly see the new message.
## Outbound SMTP
Use [ZoneMTA](https://github.com/zone-eu/zone-mta) with the [ZoneMTA-WildDuck](https://github.com/wildduck-email/zonemta-wildduck) plugin. This gives you an outbound SMTP server that uses Wild Duck accounts for authentication.
## Outbound SMTP
Use [Haraka](http://haraka.github.io/) with [queue/lmtp](http://haraka.github.io/manual/plugins/queue/lmtp.html) plugin. Wild Duck specific recipient processing plugin coming soon!
## License
Wild Duck Mail Agent is licensed under the [European Union Public License 1.1](http://ec.europa.eu/idabc/eupl.html).

12
api.js
View file

@ -852,7 +852,7 @@ server.get('/mailbox/:id', (req, res, next) => {
db.database.collection('messages').find(query, {
uid: true,
mailbox: true,
internaldate: true,
idate: true,
headers: true,
hasAttachments: true,
intro: true
@ -898,7 +898,7 @@ server.get('/mailbox/:id', (req, res, next) => {
messages: messages.map(message => {
let response = {
id: message._id,
date: message.internaldate,
date: message.idate,
hasAttachments: message.hasAttachments,
intro: message.intro
};
@ -960,7 +960,7 @@ server.get('/message/:id', (req, res, next) => {
html: true,
text: true,
attachments: true,
internaldate: true,
idate: true,
flags: true
}, (err, message) => {
if (err) {
@ -984,7 +984,7 @@ server.get('/message/:id', (req, res, next) => {
id,
mailbox: message.mailbox,
headers: message.headers,
date: message.internaldate,
date: message.idate,
flags: message.flags,
text: message.text,
html: message.html,
@ -1189,6 +1189,10 @@ server.del('/message/:id', (req, res, next) => {
});
module.exports = done => {
if (!config.imap.enabled) {
return setImmediate(() => done(null, false));
}
let started = false;
messageHandler = new MessageHandler(db.database);

5
config/default.js
View file

@ -23,6 +23,7 @@ module.exports = {
redis: 'redis://127.0.0.1:6379/3',
imap: {
enabled: true,
port: 9993,
host: '127.0.0.1',
// If certificate path is not defined, use built-in self-signed certs
@ -56,7 +57,9 @@ module.exports = {
},
api: {
port: 8080
enabled: true,
port: 8080,
host: '0.0.0.0'
},
// if this header exists and starts with yes then the message is treated as spam

11
imap-core/lib/commands/logout.js
View file

@ -1,5 +1,14 @@
'use strict';
let quotes = [
'All dreams are but another reality. Never forget...',
'Oh boy, oh boy, oh boy...',
'Cut the dramatics, would yeh, and follow me!',
'Oh ho ho ho, duck hunters is da cwaziest peoples! Ha ha ha.',
'Well, that makes sense. Send a bird to catch a cat!',
'Piccobello!'
];
module.exports = {
handler(command) {
this.session.selected = this.selected = false;
@ -7,7 +16,7 @@ module.exports = {
this.updateNotificationListener(() => {
this.send('* BYE Logout requested');
this.send(command.tag + ' OK All dreams are but another reality. Never forget...');
this.send(command.tag + ' OK ' + quotes[Math.floor(Math.random() * quotes.length)]);
this.close();
});
}

56
imap-core/lib/imap-connection.js
View file

@ -109,7 +109,10 @@ class IMAPConnection extends EventEmitter {
this._startSession();
this._server.logger.info('[%s] Connection from %s', this.id, this.clientHostname);
this._server.logger.info({
tnx: 'connect',
cid: this.id
}, '[%s] Connection from %s', this.id, this.clientHostname);
this.send('* OK ' + (this._server.options.id && this._server.options.id.name || packageInfo.name) + ' ready');
});
}
@ -123,7 +126,10 @@ class IMAPConnection extends EventEmitter {
send(payload, callback) {
if (this._socket && this._socket.writable) {
this[!this.compression ? '_socket' : '_deflate'].write(payload + '\r\n', 'binary', callback);
this._server.logger.debug('[%s] S:', this.id, payload);
this._server.logger.debug({
tnx: 'send',
cid: this.id
}, '[%s] S:', this.id, payload);
}
}
@ -158,7 +164,10 @@ class IMAPConnection extends EventEmitter {
* @event
*/
_onEnd() {
this._server.logger.info('[%s] Connection END', this.id);
this._server.logger.info({
tnx: 'close',
cid: this.id
}, '[%s] Connection END', this.id);
if (!this._closed) {
this._onClose();
}
@ -203,7 +212,10 @@ class IMAPConnection extends EventEmitter {
this._closed = true;
this._closing = false;
this._server.logger.info('[%s] Connection closed to %s', this.id, this.clientHostname);
this._server.logger.info({
tnx: 'close',
cid: this.id
}, '[%s] Connection closed to %s', this.id, this.clientHostname);
}
/**
@ -218,7 +230,10 @@ class IMAPConnection extends EventEmitter {
return;
}
this._server.logger.error('[%s] %s', this.id, err.message);
this._server.logger.error({
err,
cid: this.id
}, '[%s] %s', this.id, err.message);
this.emit('error', err);
}
@ -228,7 +243,10 @@ class IMAPConnection extends EventEmitter {
* @event
*/
_onTimeout() {
this._server.logger.info('[%s] Connection TIMEOUT', this.id);
this._server.logger.info({
tnx: 'connection',
cid: this.id
}, '[%s] Connection TIMEOUT', this.id);
if (this.idling) {
return; // ignore timeouts when IDLEing
}
@ -354,7 +372,11 @@ class IMAPConnection extends EventEmitter {
listenerData.lock = false;
if (err) {
this._server.logger.info('[%s] Notification Error: %s', this.id, err.message);
this._server.logger.info({
err,
tnx: 'updates',
cid: this.id
}, '[%s] Notification Error: %s', this.id, err.message);
return;
}
@ -398,7 +420,10 @@ class IMAPConnection extends EventEmitter {
let existsResponse;
// show notifications
this._server.logger.info('[%s] Pending notifications: %s', this.id, this.selected.notifications.length);
this._server.logger.info({
tnx: 'notifications',
cid: this.id
}, '[%s] Pending notifications: %s', this.id, this.selected.notifications.length);
// find UIDs that are both added and removed
let added = new Set(); // added UIDs
@ -446,13 +471,19 @@ class IMAPConnection extends EventEmitter {
this.selected.modifyIndex = update.modseq;
}
this._server.logger.info('[%s] Processing notification: %s', this.id, JSON.stringify(update));
this._server.logger.info({
tnx: 'notifications',
cid: this.id
}, '[%s] Processing notification: %s', this.id, JSON.stringify(update));
if (update.ignore === this.id) {
continue; // skip this
}
this._server.logger.info('[%s] UIDS: %s', this.id, this.selected.uidList.length);
this._server.logger.info({
tnx: 'notifications',
cid: this.id
}, '[%s] UIDS: %s', this.id, this.selected.uidList.length);
switch (update.command) {
case 'EXISTS':
@ -468,7 +499,10 @@ class IMAPConnection extends EventEmitter {
case 'EXPUNGE':
{
let seq = (this.selected.uidList || []).indexOf(update.uid);
this._server.logger.info('[%s] EXPUNGE %s', this.id, seq);
this._server.logger.info({
tnx: 'expunge',
cid: this.id
}, '[%s] EXPUNGE %s', this.id, seq);
if (seq >= 0) {
let output = this.formatResponse('EXPUNGE', update.uid);
this.writeStream.write(output);

6
imap-core/lib/imap-tools.js
View file

@ -496,10 +496,10 @@ module.exports.getQueryResponse = function (query, message, options) {
break;
case 'internaldate':
if (!message.internaldate) {
message.internaldate = new Date();
if (!message.idate) {
message.idate = new Date();
}
value = message.internaldate;
value = message.idate;
break;
case 'bodystructure':

8
imap-core/lib/search.js
View file

@ -21,11 +21,11 @@ let queryHandlers = {
internaldate(message, query, callback) {
switch (query.operator) {
case '<':
return callback(null, getShortDate(message.internaldate) < getShortDate(query.value));
return callback(null, getShortDate(message.idate) < getShortDate(query.value));
case '=':
return callback(null, getShortDate(message.internaldate) === getShortDate(query.value));
return callback(null, getShortDate(message.idate) === getShortDate(query.value));
case '>=':
return callback(null, getShortDate(message.internaldate) >= getShortDate(query.value));
return callback(null, getShortDate(message.idate) >= getShortDate(query.value));
}
return callback(null, false);
},
@ -40,7 +40,7 @@ let queryHandlers = {
if (!mimeTree) {
mimeTree = indexer.parseMimeTree(message.raw);
}
date = mimeTree.parsedHeader.date || message.internaldate;
date = mimeTree.parsedHeader.date || message.idate;
}
switch (query.operator) {

12
imap-core/test/test-server.js
View file

@ -22,19 +22,19 @@ module.exports = function (options) {
uid: 45,
flags: [],
modseq: 100,
internaldate: new Date('14-Sep-2013 21:22:28 -0300'),
idate: new Date('14-Sep-2013 21:22:28 -0300'),
mimeTree: parseMimeTree(new Buffer('from: sender@example.com\r\nto: to@example.com\r\ncc: cc@example.com\r\nsubject: test\r\n\r\nzzzz'))
}, {
uid: 49,
flags: ['\\Seen'],
internaldate: new Date(),
idate: new Date(),
modseq: 5000,
mimeTree: parseMimeTree(fs.readFileSync(__dirname + '/fixtures/ryan_finnie_mime_torture.eml'))
}, {
uid: 50,
flags: ['\\Seen'],
modseq: 45,
internaldate: new Date(),
idate: new Date(),
mimeTree: parseMimeTree('MIME-Version: 1.0\r\n' +
'From: andris@kreata.ee\r\n' +
'To: andris@tr.ee\r\n' +
@ -71,18 +71,18 @@ module.exports = function (options) {
uid: 52,
flags: [],
modseq: 4,
internaldate: new Date(),
idate: new Date(),
mimeTree: parseMimeTree('from: sender@example.com\r\nto: to@example.com\r\ncc: cc@example.com\r\nsubject: test\r\n\r\nHello World!')
}, {
uid: 53,
flags: [],
modseq: 5,
internaldate: new Date()
idate: new Date()
}, {
uid: 60,
flags: [],
modseq: 6,
internaldate: new Date()
idate: new Date()
}],
journal: []
}, {

8
imap.js
View file

@ -1033,7 +1033,7 @@ server.onFetch = function (path, options, session, callback) {
let projection = {
uid: true,
modseq: true,
internaldate: true,
idate: true,
flags: true,
envelope: true,
bodystructure: true,
@ -1403,7 +1403,7 @@ server.onSearch = function (path, options, session, callback) {
};
entry = {
internaldate: !ne ? entry : {
idate: !ne ? entry : {
$not: entry
}
};
@ -1587,6 +1587,10 @@ server.onGetQuota = function (quotaRoot, session, callback) {
};
module.exports = done => {
if (!config.imap.enabled) {
return setImmediate(() => done(null, false));
}
let start = () => {
messageHandler = new MessageHandler(db.database);

4
indexes.json
View file

@ -91,10 +91,10 @@
"modseq": 1
}
}, {
"name": "by_internaldate",
"name": "by_idate",
"key": {
"mailbox": 1,
"internaldate": 1
"idate": 1
}
}, {
"name": "by_hdate",

132
lib/message-handler.js
View file

@ -64,56 +64,18 @@ class MessageHandler {
// TODO: Refactor into smaller pieces
add(options, callback) {
let id = new ObjectID();
let prepared = options.prepared || this.prepareMessage(options);
let mimeTree = this.indexer.parseMimeTree(options.raw);
let size = this.indexer.getSize(mimeTree);
let bodystructure = this.indexer.getBodyStructure(mimeTree);
let envelope = this.indexer.getEnvelope(mimeTree);
let internaldate = options.date && new Date(options.date) || new Date();
let hdate = mimeTree.parsedHeader.date && new Date(mimeTree.parsedHeader.date) || false;
let flags = [].concat(options.flags || []);
if (!hdate || hdate.toString() === 'Invalid Date') {
hdate = internaldate;
}
let msgid = envelope[9] || ('<' + uuidV1() + '@wildduck.email>');
let headers = (mimeTree.header || []).map(line => {
line = Buffer.from(line, 'binary').toString();
let key = line.substr(0, line.indexOf(':')).trim().toLowerCase();
let value = line.substr(line.indexOf(':') + 1).trim().toLowerCase().replace(/\s*\r?\n\s*/g, ' ');
try {
value = libmime.decodeWords(value);
} catch (E) {
// ignore
}
// trim long values as mongodb indexed fields can not be too long
if (Buffer.byteLength(key, 'utf-8') >= 255) {
key = Buffer.from(key).slice(0, 255).toString();
key = key.substr(0, key.length - 4);
}
if (Buffer.byteLength(value, 'utf-8') >= 880) {
// value exceeds MongoDB max indexed value length
value = Buffer.from(value).slice(0, 880).toString();
// remove last 4 chars to be sure we do not have any incomplete unicode sequences
value = value.substr(0, value.length - 4);
}
return {
key,
value
};
});
let id = prepared.id;
let mimeTree = prepared.mimeTree;
let size = prepared.size;
let bodystructure = prepared.bodystructure;
let envelope = prepared.envelope;
let idate = prepared.idate;
let hdate = prepared.hdate;
let flags = prepared.flags;
let msgid = prepared.msgid;
let headers = prepared.headers;
this.getMailbox(options, (err, mailbox) => {
if (err) {
@ -251,7 +213,7 @@ class MessageHandler {
let message = {
_id: id,
internaldate,
idate,
hdate,
flags,
size,
@ -665,6 +627,76 @@ class MessageHandler {
});
});
}
generateIndexedHeaders(headersArray) {
return (headersArray || []).map(line => {
line = Buffer.from(line, 'binary').toString();
let key = line.substr(0, line.indexOf(':')).trim().toLowerCase();
let value = line.substr(line.indexOf(':') + 1).trim().toLowerCase().replace(/\s*\r?\n\s*/g, ' ');
try {
value = libmime.decodeWords(value);
} catch (E) {
// ignore
}
// trim long values as mongodb indexed fields can not be too long
if (Buffer.byteLength(key, 'utf-8') >= 255) {
key = Buffer.from(key).slice(0, 255).toString();
key = key.substr(0, key.length - 4);
}
if (Buffer.byteLength(value, 'utf-8') >= 880) {
// value exceeds MongoDB max indexed value length
value = Buffer.from(value).slice(0, 880).toString();
// remove last 4 chars to be sure we do not have any incomplete unicode sequences
value = value.substr(0, value.length - 4);
}
return {
key,
value
};
});
}
prepareMessage(options) {
let id = new ObjectID();
let mimeTree = this.indexer.parseMimeTree(options.raw);
let size = this.indexer.getSize(mimeTree);
let bodystructure = this.indexer.getBodyStructure(mimeTree);
let envelope = this.indexer.getEnvelope(mimeTree);
let idate = options.date && new Date(options.date) || new Date();
let hdate = mimeTree.parsedHeader.date && new Date(mimeTree.parsedHeader.date) || false;
let flags = [].concat(options.flags || []);
if (!hdate || hdate.toString() === 'Invalid Date') {
hdate = idate;
}
let msgid = envelope[9] || ('<' + uuidV1() + '@wildduck.email>');
let headers = this.generateIndexedHeaders(mimeTree.header);
return {
id,
mimeTree,
size,
bodystructure,
envelope,
idate,
hdate,
flags,
msgid,
headers
};
}
}
module.exports = MessageHandler;

48
lmtp.js
View file

@ -7,7 +7,6 @@ const log = require('npmlog');
const SMTPServer = require('smtp-server').SMTPServer;
const tools = require('./lib/tools');
const MessageHandler = require('./lib/message-handler');
const MessageSplitter = require('./lib/message-splitter');
const db = require('./lib/db');
const fs = require('fs');
@ -96,11 +95,9 @@ const serverOptions = {
let chunks = [];
let chunklen = 0;
let splitter = new MessageSplitter();
splitter.on('readable', () => {
stream.on('readable', () => {
let chunk;
while ((chunk = splitter.read()) !== null) {
while ((chunk = stream.read()) !== null) {
chunks.push(chunk);
chunklen += chunk.length;
}
@ -111,30 +108,16 @@ const serverOptions = {
callback(new Error('Error reading from stream'));
});
splitter.once('end', () => {
chunks.unshift(splitter.rawHeaders);
chunklen += splitter.rawHeaders.length;
stream.once('end', () => {
let isSpam = false;
let spamHeader = config.spamHeader && config.spamHeader.toLowerCase();
if (Array.isArray(splitter.headers)) {
for (let i = splitter.headers.length - 1; i >= 0; i--) {
let header = splitter.headers[i];
// check if the header is used for detecting spam
if (spamHeader === header.key) {
let value = header.line.substr(header.line.indexOf(':') + 1).trim();
if (/^yes\b/i.test(value)) {
isSpam = true;
}
}
}
}
let isSpam = false;
let responses = [];
let users = session.users;
let stored = 0;
let storeNext = () => {
if (stored >= users.length) {
return callback(null, responses.map(r => r.response));
@ -159,9 +142,24 @@ const serverOptions = {
chunks.unshift(header);
chunklen += header.length;
let prepared = messageHandler.prepareMessage({raw: Buffer.concat(chunks, chunklen)});
let mailboxQueryKey = 'path';
let mailboxQueryValue = 'INBOX';
// apply filters
if (spamHeader) {
for (let i = prepared.headers.length - 1; i >= 0; i--) {
let header = prepared.headers[i];
// check if the header is used for detecting spam
if (spamHeader === header.key) {
if (/^yes\b/i.test(header.value)) {
isSpam = true;
}
}
}
}
if (isSpam) {
mailboxQueryKey = 'specialUse';
mailboxQueryValue = '\\Junk';
@ -171,6 +169,8 @@ const serverOptions = {
user,
[mailboxQueryKey]: mailboxQueryValue,
prepared,
meta: {
source: 'LMTP',
from: tools.normalizeAddress(session.envelope.mailFrom && session.envelope.mailFrom.address || ''),
@ -188,8 +188,6 @@ const serverOptions = {
skipExisting: true
};
messageOptions.raw = Buffer.concat(chunks, chunklen);
messageHandler.add(messageOptions, (err, inserted, info) => {
// remove Delivered-To
@ -208,8 +206,6 @@ const serverOptions = {
storeNext();
});
stream.pipe(splitter);
}
};

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "wildduck",
"version": "1.0.16",
"version": "1.0.17",
"description": "IMAP server built with Node.js and MongoDB",
"main": "server.js",
"scripts": {