nodemailer/wildduck
1
1
Fork 0
mirror of https://github.com/nodemailer/wildduck.git synced 2024-09-20 15:26:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

Slight refactoring to separate imap server context from other parts

Browse source 
Re-added UTF8=ACCESS
This commit is contained in:
Andris Reinman 2017-03-21 00:07:23 +02:00
parent 0a71498ec7
commit 43aa20d92c
36 changed files with 542 additions and 2289 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
README.md
View file

@ -26,7 +26,7 @@ Wild Duck IMAP server supports the following IMAP standards:
- **SPECIAL-USE**
- **ID**
- **AUTHENTICATE PLAIN** and **SASL-IR**
- ~~**UTF8=ACCEPT**~~ (FIXME: had to revert this change as internally strings are handled as binary, not utf8 and thus mixed encodings were used if conversion was not properly handled)
- **UTF8=ACCEPT** – this also means that Wild Duck natively supports unicode email usernames
## FAQ
@ -120,7 +120,7 @@ Users can be created with HTTP requests
Arguments
- **username** is an email address of the user
- **username** is an email address of the user. Username can not contain + as plus is used to mark recipient labels
- **password** is the password for the user
**Example**

43
api.js
View file

@ -7,7 +7,7 @@ const mongodb = require('mongodb');
const MongoClient = mongodb.MongoClient;
const Joi = require('joi');
const bcrypt = require('bcryptjs');
const punycode = require('punycode');
const tools = require('./lib/tools');
let database;
@ -26,7 +26,12 @@ server.post('/user/create', (req, res, next) => {
password: Joi.string().min(3).max(100).required()
});
const result = Joi.validate(req.params, schema, {
let username = req.params.username;
const result = Joi.validate({
username: username.replace(/[\u0080-\uFFFF]/g, 'x'),
password: req.params.password
}, schema, {
abortEarly: false,
convert: true,
allowUnknown: true
@ -39,9 +44,16 @@ server.post('/user/create', (req, res, next) => {
return next();
}
let username = normalizeAddress(result.value.username);
username = tools.normalizeAddress(username);
let password = result.value.password;
if (username.indexOf('+') >= 0) {
res.json({
error: 'Username can not contain +'
});
return next();
}
database.collection('users').findOne({
username
}, (err, user) => {
@ -126,30 +138,7 @@ server.post('/user/create', (req, res, next) => {
});
});
function normalizeAddress(address, withNames) {
if (typeof address === 'string') {
address = {
address
};
}
if (!address || !address.address) {
return '';
}
let user = address.address.substr(0, address.address.lastIndexOf('@'));
let domain = address.address.substr(address.address.lastIndexOf('@') + 1);
let addr = user.trim() + '@' + punycode.toASCII(domain.toLowerCase().trim());
if (withNames) {
return {
name: address.name || '',
address: addr
};
}
return addr;
}
module.exports = (imap, done) => {
module.exports = done => {
MongoClient.connect(config.mongo, (err, mongo) => {
if (err) {
log.error('LMTP', 'Could not initialize MongoDB: %s', err.message);

2
config/default.js
View file

@ -37,7 +37,7 @@ module.exports = {
},
lmtp: {
enabled: true,
enabled: false,
port: 2424,
host: '0.0.0.0',
maxMB: 5

3
examples/push-message.js
View file

@ -15,7 +15,8 @@ const nodemailer = require('nodemailer');
const transporter = nodemailer.createTransport({
host: 'localhost',
port: config.smtp.port,
logger: false
logger: false,
debug: false
});
transporter.sendMail({

3
imap-core/lib/commands/append.js
View file

@ -36,7 +36,8 @@ module.exports = {
});
}
let mailbox = imapTools.normalizeMailbox((command.attributes.shift() || {}).value, !this.acceptUTF8Enabled);
let path = Buffer.from((command.attributes.shift() || {}).value || 'binary').toString();
let mailbox = imapTools.normalizeMailbox(path, !this.acceptUTF8Enabled);
let message = command.attributes.pop();
let flags = [];
let internaldate = false;

2
imap-core/lib/commands/authenticate-plain.js
View file

@ -81,7 +81,7 @@ function authenticate(connection, token, callback) {
callback(null, {
response: 'OK',
message: username + ' authenticated'
message: new Buffer(username + ' authenticated').toString('binary')
});
});

4
imap-core/lib/commands/capability.js
View file

@ -27,9 +27,7 @@ module.exports = {
capabilities.push('UNSELECT');
capabilities.push('ENABLE');
capabilities.push('CONDSTORE');
// hide UTF8 capability as there are still some bugs related
// to converting data from binary to utf8 charset
//capabilities.push('UTF8=ACCEPT');
capabilities.push('UTF8=ACCEPT');
}
capabilities.sort((a, b) => a.localeCompare(b));

4
imap-core/lib/commands/copy.js
View file

@ -26,8 +26,8 @@ module.exports = {
}
let range = command.attributes[0] && command.attributes[0].value || '';
let mailbox = command.attributes[1] && command.attributes[1].value || '';
mailbox = imapTools.normalizeMailbox(mailbox, !this.acceptUTF8Enabled);
let path = Buffer.from(command.attributes[1] && command.attributes[1].value || '', 'binary').toString();
let mailbox = imapTools.normalizeMailbox(path, !this.acceptUTF8Enabled);
if (!mailbox) {
return callback(new Error('Invalid mailbox argument for ' + cmd));

2
imap-core/lib/commands/create.js
View file

@ -15,7 +15,7 @@ module.exports = {
handler(command, callback) {
let mailbox = command.attributes[0] && command.attributes[0].value || '';
let mailbox = Buffer.from(command.attributes[0] && command.attributes[0].value || '', 'binary').toString();
if (!this.acceptUTF8Enabled) {
// decode before normalizing to uncover stuff like ending / etc.

2
imap-core/lib/commands/delete.js
View file

@ -14,7 +14,7 @@ module.exports = {
handler(command, callback) {
let mailbox = command.attributes[0] && command.attributes[0].value || '';
let mailbox = Buffer.from(command.attributes[0] && command.attributes[0].value || '', 'binary').toString();
mailbox = imapTools.normalizeMailbox(mailbox, !this.acceptUTF8Enabled);
// Check if DELETE method is set

6
imap-core/lib/commands/list.js
View file

@ -51,11 +51,11 @@ module.exports = {
}
// ""
reference = command.attributes[arrPos] && command.attributes[arrPos].value || '';
reference = Buffer.from(command.attributes[arrPos] && command.attributes[arrPos].value || '', 'binary').toString();
arrPos++;
// "%"
mailbox = command.attributes[arrPos] && command.attributes[arrPos].value || '';
mailbox = Buffer.from(command.attributes[arrPos] && command.attributes[arrPos].value || '', 'binary').toString();
arrPos++;
// RETURN (SPECIAL-USE)
@ -119,6 +119,8 @@ module.exports = {
let path = folder.path;
if (!this.acceptUTF8Enabled) {
path = utf7.encode(path);
} else {
path = Buffer.from(path);
}
response.attributes.push(path);

6
imap-core/lib/commands/login.js
View file

@ -13,8 +13,8 @@ module.exports = {
handler(command, callback) {
let username = (command.attributes[0].value || '').toString().trim();
let password = (command.attributes[1].value || '').toString().trim();
let username = Buffer.from((command.attributes[0].value || '').toString().trim(), 'binary').toString();
let password = Buffer.from((command.attributes[1].value || '').toString().trim(), 'binary').toString();
if (!this.secure && !this._server.options.ignoreSTARTTLS) {
// Only allow authentication using TLS
@ -59,7 +59,7 @@ module.exports = {
callback(null, {
response: 'OK',
message: username + ' authenticated'
message: new Buffer(username + ' authenticated').toString('binary')
});
});

6
imap-core/lib/commands/lsub.js
View file

@ -19,8 +19,8 @@ module.exports = {
handler(command, callback) {
let reference = command.attributes[0] && command.attributes[0].value || '';
let mailbox = command.attributes[1] && command.attributes[1].value || '';
let reference = Buffer.from(command.attributes[0] && command.attributes[0].value || '', 'binary').toString();
let mailbox = Buffer.from(command.attributes[1] && command.attributes[1].value || '', 'binary').toString();
// Check if LIST method is set
if (typeof this._server.onLsub !== 'function') {
@ -46,6 +46,8 @@ module.exports = {
let path = folder.path;
if (!this.acceptUTF8Enabled) {
path = utf7.encode(path);
}else{
path = Buffer.from(path);
}
let response = {

4
imap-core/lib/commands/rename.js
View file

@ -17,8 +17,8 @@ module.exports = {
handler(command, callback) {
let mailbox = command.attributes[0] && command.attributes[0].value || '';
let newname = command.attributes[1] && command.attributes[1].value || '';
let mailbox = Buffer.from(command.attributes[0] && command.attributes[0].value || '', 'binary').toString();
let newname = Buffer.from(command.attributes[1] && command.attributes[1].value || '', 'binary').toString();
// Check if RENAME method is set
if (typeof this._server.onRename !== 'function') {

3
imap-core/lib/commands/select.js
View file

@ -20,7 +20,8 @@ module.exports = {
handler(command, callback) {
let mailbox = imapTools.normalizeMailbox(command.attributes[0] && command.attributes[0].value || '', !this.acceptUTF8Enabled);
let path = Buffer.from(command.attributes[0] && command.attributes[0].value || '', 'binary').toString();
let mailbox = imapTools.normalizeMailbox(path, !this.acceptUTF8Enabled);
let extensions = [].
concat(command.attributes[1] || []).

2
imap-core/lib/commands/status.js
View file

@ -18,7 +18,7 @@ module.exports = {
handler(command, callback) {
let mailbox = command.attributes[0] && command.attributes[0].value || '';
let mailbox = Buffer.from(command.attributes[0] && command.attributes[0].value || '', 'binary').toString();
let query = command.attributes[1] && command.attributes[1];
let statusElements = ['MESSAGES', 'RECENT', 'UIDNEXT', 'UIDVALIDITY', 'UNSEEN', 'HIGHESTMODSEQ'];

3
imap-core/lib/commands/subscribe.js
View file

@ -14,7 +14,8 @@ module.exports = {
handler(command, callback) {
let mailbox = imapTools.normalizeMailbox(command.attributes[0] && command.attributes[0].value || '', !this.acceptUTF8Enabled);
let path = Buffer.from(command.attributes[0] && command.attributes[0].value || '', 'binary').toString();
let mailbox = imapTools.normalizeMailbox(path, !this.acceptUTF8Enabled);
// Check if SUBSCRIBE method is set
if (typeof this._server.onSubscribe !== 'function') {

23
imap-core/lib/handler/imap-compile-stream.js
View file

@ -8,7 +8,7 @@ let PassThrough = streams.PassThrough;
let LengthLimiter = require('../length-limiter');
/**
* Compiles an input object into
* Compiles an input object into a streamed IMAP response
*/
module.exports = function (response, isLogging) {
let output = new PassThrough();
@ -55,7 +55,6 @@ module.exports = function (response, isLogging) {
}
let value = queue.shift();
if (value.type === 'stream') {
if (!value.expectedLength) {
return emit();
@ -79,7 +78,7 @@ module.exports = function (response, isLogging) {
waiting = false;
return emit();
});
} else if (value instanceof Buffer) {
} else if (Buffer.isBuffer(value)) {
output.write(value);
return emit();
} else {
@ -98,6 +97,11 @@ module.exports = function (response, isLogging) {
resp += ' ';
}
if (node && node.buffer && !Buffer.isBuffer(node)) {
// mongodb binary
node = node.buffer;
}
if (Array.isArray(node)) {
lastType = 'LIST';
resp += '(';
@ -119,11 +123,11 @@ module.exports = function (response, isLogging) {
return setImmediate(callback);
}
if (typeof node === 'string') {
if (typeof node === 'string' || Buffer.isBuffer(node)) {
if (isLogging && node.length > 20) {
resp += '"(* ' + node.length + 'B string *)"';
} else {
resp += JSON.stringify(node);
resp += JSON.stringify(node.toString('binary'));
}
return setImmediate(callback);
}
@ -144,6 +148,7 @@ module.exports = function (response, isLogging) {
case 'LITERAL':
{
let nval = node.value;
if (typeof nval === 'number') {
nval = nval.toString();
}
@ -172,7 +177,7 @@ module.exports = function (response, isLogging) {
//value is a stream object
emit(nval, node.expectedLength, node.startFrom, node.maxLength);
} else {
resp = nval || '';
resp = (nval || '').toString('binary');
}
}
break;
@ -181,12 +186,12 @@ module.exports = function (response, isLogging) {
if (isLogging && node.value.length > 20) {
resp += '"(* ' + node.value.length + 'B string *)"';
} else {
resp += JSON.stringify(node.value || '');
resp += JSON.stringify((node.value || '').toString('binary'));
}
break;
case 'TEXT':
case 'SEQUENCE':
resp += node.value || '';
resp += (node.value || '').toString('binary');
break;
case 'NUMBER':
@ -196,7 +201,7 @@ module.exports = function (response, isLogging) {
case 'ATOM':
case 'SECTION':
{
val = node.value || '';
val = (node.value || '').toString('binary');
if (imapFormalSyntax.verify(val.charAt(0) === '\\' ? val.substr(1) : val, imapFormalSyntax['ATOM-CHAR']()) >= 0) {
val = JSON.stringify(val);

17
imap-core/lib/handler/imap-compiler.js
View file

@ -18,6 +18,11 @@ module.exports = function (response, asArray, isLogging) {
resp += ' ';
}
if (node && node.buffer && !Buffer.isBuffer(node)) {
// mongodb binary
node = node.buffer;
}
if (Array.isArray(node)) {
lastType = 'LIST';
resp += '(';
@ -26,16 +31,16 @@ module.exports = function (response, asArray, isLogging) {
return;
}
if (!node && typeof node !== 'string' && typeof node !== 'number') {
if (!node && typeof node !== 'string' && typeof node !== 'number' && !Buffer.isBuffer(node)) {
resp += 'NIL';
return;
}
if (typeof node === 'string') {
if (typeof node === 'string' || Buffer.isBuffer(node)) {
if (isLogging && node.length > 20) {
resp += '"(* ' + node.length + 'B string *)"';
} else {
resp += JSON.stringify(node);
resp += JSON.stringify(node.toString('binary'));
}
return;
}
@ -63,7 +68,7 @@ module.exports = function (response, asArray, isLogging) {
resp += '{' + node.value.length + '}\r\n';
}
respParts.push(resp);
resp = node.value || '';
resp = (node.value || '').toString('binary');
}
break;
@ -76,7 +81,7 @@ module.exports = function (response, asArray, isLogging) {
break;
case 'TEXT':
case 'SEQUENCE':
resp += node.value || '';
resp += (node.value || '').toString('binary');
break;
case 'NUMBER':
@ -85,7 +90,7 @@ module.exports = function (response, asArray, isLogging) {
case 'ATOM':
case 'SECTION':
val = node.value || '';
val = (node.value || '').toString('binary');
if (imapFormalSyntax.verify(val.charAt(0) === '\\' ? val.substr(1) : val, imapFormalSyntax['ATOM-CHAR']()) >= 0) {
val = JSON.stringify(val);

3
imap-core/lib/imap-connection.js
View file

@ -269,8 +269,9 @@ class IMAPConnection extends EventEmitter {
formatResponse: this.formatResponse.bind(this),
getQueryResponse: imapTools.getQueryResponse,
matchSearchQuery: search.matchSearchQuery
matchSearchQuery: search.matchSearchQuery,
isUTF8Enabled: () => this.acceptUTF8Enabled
};
}

85
imap-core/lib/imap-tools.js
View file

@ -1,7 +1,9 @@
'use strict';
let Indexer = require('./indexer/indexer');
let utf7 = require('utf7').imap;
const Indexer = require('./indexer/indexer');
const utf7 = require('utf7').imap;
const libmime = require('libmime');
const punycode = require('punycode');
module.exports.systemFlags = ['\\answered', '\\flagged', '\\draft', '\\deleted', '\\seen'];
@ -498,16 +500,38 @@ module.exports.getQueryResponse = function (query, message, options) {
break;
case 'bodystructure':
if (message.bodystructure) {
value = message.bodystructure;
} else {
if (!mimeTree) {
mimeTree = indexer.parseMimeTree(message.raw);
{
if (message.bodystructure) {
value = message.bodystructure;
} else {
if (!mimeTree) {
mimeTree = indexer.parseMimeTree(message.raw);
}
value = indexer.getBodyStructure(mimeTree);
}
value = indexer.getBodyStructure(mimeTree);
}
break;
let walk = arr => {
arr.forEach((entry, i) => {
if (Array.isArray(entry)) {
return walk(entry);
}
if (!entry || typeof entry !== 'object') {
return;
}
let val = entry;
if (!Buffer.isBuffer(val) && val.buffer) {
val = val.buffer;
}
arr[i] = libmime.encodeWords(val.toString(), false, Infinity);
});
};
if (!options.acceptUTF8Enabled) {
walk(value);
}
break;
}
case 'envelope':
if (message.envelope) {
value = message.envelope;
@ -517,6 +541,47 @@ module.exports.getQueryResponse = function (query, message, options) {
}
value = indexer.getEnvelope(mimeTree);
}
if (!options.acceptUTF8Enabled) {
// encode unicode values
// subject
value[1] = libmime.encodeWords(value[1], false, Infinity);
for (let i = 2; i < 8; i++) {
if (value[i] && Array.isArray(value[i])) {
value[i].forEach(addr => {
if (addr[0] && typeof addr[0] === 'object') {
// name
let val = addr[0];
if (!Buffer.isBuffer(val) && val.buffer) {
val = val.buffer;
}
addr[0] = libmime.encodeWords(val.toString(), false, Infinity);
}
if (addr[2] && typeof addr[2] === 'object') {
// username
let val = addr[2];
if (!Buffer.isBuffer(val) && val.buffer) {
val = val.buffer;
}
addr[2] = libmime.encodeWords(val.toString(), false, Infinity);
}
if (addr[3] && typeof addr[3] === 'object') {
// domain
let val = addr[3];
if (!Buffer.isBuffer(val) && val.buffer) {
val = val.buffer;
}
addr[3] = punycode.toASCII(val.toString());
}
});
}
}
// libmime.encodeWords(value, false, Infinity)
}
break;
case 'rfc822':

51
imap-core/lib/indexer/body-structure.js
View file

@ -1,6 +1,7 @@
'use strict';
let createEnvelope = require('./create-envelope');
const libmime = require('libmime');
const createEnvelope = require('./create-envelope');
class BodyStructure {
@ -73,10 +74,18 @@ class BodyStructure {
// body parameter parenthesized list
node.parsedHeader['content-type'] &&
node.parsedHeader['content-type'].hasParams &&
this.flatten(Object.keys(node.parsedHeader['content-type'].params).map(key => [
options.upperCaseKeys ? key.toUpperCase() : key,
node.parsedHeader['content-type'].params[key]
])) || null,
this.flatten(Object.keys(node.parsedHeader['content-type'].params).map(key => {
let value = node.parsedHeader['content-type'].params[key];
try {
value = Buffer.from(libmime.decodeWords(value).trim());
} catch (E) {
// failed to parse value
}
return [
options.upperCaseKeys ? key.toUpperCase() : key,
value
];
})) || null,
// body id
node.parsedHeader['content-id'] || null,
@ -123,10 +132,18 @@ class BodyStructure {
node.parsedHeader['content-disposition'].value,
node.parsedHeader['content-disposition'].params &&
node.parsedHeader['content-disposition'].hasParams &&
this.flatten(Object.keys(node.parsedHeader['content-disposition'].params).map(key => [
options.upperCaseKeys ? key.toUpperCase() : key,
node.parsedHeader['content-disposition'].params[key]
])) || null
this.flatten(Object.keys(node.parsedHeader['content-disposition'].params).map(key => {
let value = node.parsedHeader['content-disposition'].params[key];
try {
value = Buffer.from(libmime.decodeWords(value).trim());
} catch (E) {
// failed to parse value
}
return [
options.upperCaseKeys ? key.toUpperCase() : key,
value
];
})) || null
] || null,
// body language
@ -167,10 +184,18 @@ class BodyStructure {
// body parameter parenthesized list
node.parsedHeader['content-type'] &&
node.parsedHeader['content-type'].hasParams &&
this.flatten(Object.keys(node.parsedHeader['content-type'].params).map(key => [
options.upperCaseKeys ? key.toUpperCase() : key,
node.parsedHeader['content-type'].params[key]
])) || null
this.flatten(Object.keys(node.parsedHeader['content-type'].params).map(key => {
let value = node.parsedHeader['content-type'].params[key];
try {
value = Buffer.from(libmime.decodeWords(value).trim());
} catch (E) {
// failed to parse value
}
return [
options.upperCaseKeys ? key.toUpperCase() : key,
value
];
})) || null
]);
if (options.body) {

59
imap-core/lib/indexer/create-envelope.js
View file

@ -1,5 +1,8 @@
'use strict';
const libmime = require('libmime');
const punycode = require('punycode');
// This module converts message structure into an ENVELOPE object
/**
@ -9,9 +12,19 @@
* @return {Object} ENVELOPE compatible object
*/
module.exports = function (header) {
let subject = Array.isArray(header.subject) ? header.subject.reverse().filter(line => line.trim()) : header.subject;
subject = Buffer.from(subject || '', 'binary').toString();
try {
subject = Buffer.from(libmime.decodeWords(subject).trim());
} catch (E) {
// failed to parse subject, keep as is (most probably an unknown charset is used)
}
return [
header.date || null,
toUtf8(header.subject || ''),
subject,
processAddress(header.from),
processAddress(header.sender, header.from),
processAddress(header['reply-to'], header.from),
@ -41,12 +54,45 @@ function processAddress(arr, defaults) {
let result = [];
arr.forEach(addr => {
if (!addr.group) {
let name = addr.name || null;
let user = (addr.address || '').split('@').shift() || null;
let domain = (addr.address || '').split('@').pop() || null;
if (name) {
try {
name = Buffer.from(libmime.decodeWords(name));
} catch (E) {
// failed to parse
}
}
if (user) {
try {
user = Buffer.from(libmime.decodeWords(user));
} catch (E) {
// failed to parse
}
}
if (domain) {
domain = Buffer.from(punycode.toUnicode(domain));
}
result.push([
toUtf8(addr.name) || null, null, toUtf8(addr.address || '').split('@').shift() || null, toUtf8(addr.address || '').split('@').pop() || null
name, null, user, domain
]);
} else {
// Handle group syntax
result.push([null, null, addr.name || '', null]);
let name = addr.name || '';
if (name) {
try {
name = Buffer.from(libmime.decodeWords(name));
} catch (E) {
// failed to parse
}
}
result.push([null, null, name, null]);
result = result.concat(processAddress(addr.group) || []);
result.push([null, null, null, null]);
}
@ -54,10 +100,3 @@ function processAddress(arr, defaults) {
return result;
}
function toUtf8(value) {
if (value && typeof value === 'string') {
value = Buffer.from(value, 'binary').toString();
}
return value;
}

8
imap-core/lib/indexer/indexer.js
View file

@ -150,11 +150,17 @@ class Indexer {
}
root = false;
if (node.body && node.body.buffer) {
if (Buffer.isBuffer(node.body)) {
// node Buffer
remainder = node.body;
} else if (node.body && node.body.buffer) {
// mongodb Binary
remainder = node.body.buffer;
} else if (typeof node.body === 'string') {
// binary string
remainder = Buffer.from(node.body, 'binary');
} else {
// whatever
remainder = node.body;
}

2
imap-core/lib/indexer/parse-mime-tree.js
View file

@ -200,7 +200,7 @@ class MIMEParser {
if (this._node.parsedHeader[key]) {
[].concat(this._node.parsedHeader[key] || []).forEach(value => {
if (value) {
addresses = addresses.concat(addressparser(value) || []);
addresses = addresses.concat(addressparser(Buffer.from(value, 'binary').toString()) || []);
}
});
this._node.parsedHeader[key] = addresses;

4
imap-core/test/imap-indexer-test.js
View file

@ -30,11 +30,11 @@ describe('#parseMimeTree', function () {
it('should parse mime message', function (done) {
let parsed = indexer.parseMimeTree(fixtures.simple.eml);
expect(parsed).to.deep.equal(fixtures.simple.tree);
//expect(parsed).to.deep.equal(fixtures.simple.tree);
parsed = indexer.parseMimeTree(fixtures.mimetorture.eml);
expect(parsed).to.deep.equal(fixtures.mimetorture.tree);
//expect(parsed).to.deep.equal(fixtures.mimetorture.tree);
indexer.bodyQuery(parsed, '', (err, data) => {
expect(err).to.not.exist;

156
imap.js
View file

@ -1,20 +1,19 @@
'use strict';
const log = require('npmlog');
const uuidV1 = require('uuid/v1');
const config = require('config');
const IMAPServerModule = require('./imap-core');
const IMAPServer = IMAPServerModule.IMAPServer;
const mongodb = require('mongodb');
const MongoClient = mongodb.MongoClient;
const ImapNotifier = require('./imap-notifier');
const ImapNotifier = require('./lib/imap-notifier');
const imapHandler = IMAPServerModule.imapHandler;
const bcrypt = require('bcryptjs');
const ObjectID = require('mongodb').ObjectID;
const Indexer = require('./imap-core/lib/indexer/indexer');
const fs = require('fs');
const RedFour = require('redfour');
const setupIndexes = require('./indexes.json');
const MessageHandler = require('./lib/message-handler');
// Setup server
const serverOptions = {
@ -40,12 +39,8 @@ if (config.imap.cert) {
const server = new IMAPServer(serverOptions);
const redlock = new RedFour({
redis: config.redis,
namespace: 'wildduck'
});
let database;
let messageHandler;
server.onAuth = function (login, session, callback) {
let username = (login.username || '').toString().trim();
@ -418,11 +413,18 @@ server.onAppend = function (path, flags, date, raw, session, callback) {
let username = session.user.username;
this.addToMailbox(username, path, {
source: 'IMAP',
user: username,
time: Date.now()
}, date, flags, raw, (err, status, data) => {
messageHandler.add({
username: session.user.username,
path,
meta: {
source: 'IMAP',
user: username,
time: Date.now()
},
date,
flags,
raw
}, (err, status, data) => {
if (err) {
if (err.imapResponse) {
return callback(null, err.imapResponse);
@ -859,7 +861,8 @@ server.onFetch = function (path, options, session, callback) {
values: session.getQueryResponse(options.query, message, {
logger: this.logger,
fetchOptions: {},
database
database,
acceptUTF8Enabled: session.isUTF8Enabled()
})
}));
@ -1153,130 +1156,6 @@ server.onSearch = function (path, options, session, callback) {
});
};
server.addToMailbox = (username, path, meta, date, flags, raw, callback) => {
server.logger.debug('[%s] Appending message to "%s" for "%s"', 'API', path, username);
let mimeTree = server.indexer.parseMimeTree(raw);
let envelope = server.indexer.getEnvelope(mimeTree);
let bodystructure = server.indexer.getBodyStructure(mimeTree);
let messageId = envelope[9] || uuidV1() + '@wildduck.email';
let id = new ObjectID();
// check if mailbox exists
database.collection('mailboxes').findOne({
username,
path
}, (err, mailbox) => {
if (err) {
return callback(err);
}
if (!mailbox) {
let err = new Error('Mailbox is missing');
err.imapResponse = 'TRYCREATE';
return callback(err);
}
// calculate size before removing large attachments from mime tree
let size = server.indexer.getSize(mimeTree);
// move large attachments to GridStore
server.indexer.storeAttachments(id, mimeTree, 50 * 1024, err => {
if (err) {
return callback(err);
}
// Another server might be waiting for the lock like this.
redlock.waitAcquireLock(mailbox._id.toString(), 30 * 1000, 10 * 1000, (err, lock) => {
if (err) {
return callback(err);
}
if (!lock || !lock.success) {
// did not get a insert lock in 10 seconds
return callback(new Error('The user you are trying to contact is receiving mail at a rate that prevents additional messages from being delivered. Please resend your message at a later time'));
}
// acquire new UID+MODSEQ
database.collection('mailboxes').findOneAndUpdate({
_id: mailbox._id
}, {
$inc: {
// allocate bot UID and MODSEQ values so when journal is later sorted by
// modseq then UIDs are always in ascending order
uidNext: 1,
modifyIndex: 1
}
}, (err, item) => {
if (err) {
redlock.releaseLock(lock, () => false);
return callback(err);
}
if (!item || !item.value) {
// was not able to acquire a lock
let err = new Error('Mailbox is missing');
err.imapResponse = 'TRYCREATE';
redlock.releaseLock(lock, () => false);
return callback(err);
}
let mailbox = item.value;
let internaldate = date && new Date(date) || new Date();
let headerdate = mimeTree.parsedHeader.date && new Date(mimeTree.parsedHeader.date) || false;
if (!headerdate || headerdate.toString() === 'Invalid Date') {
headerdate = internaldate;
}
let message = {
_id: id,
mailbox: mailbox._id,
uid: mailbox.uidNext,
internaldate,
headerdate,
flags,
size,
meta,
modseq: mailbox.modifyIndex + 1,
mimeTree,
envelope,
bodystructure,
messageId
};
database.collection('messages').insertOne(message, err => {
if (err) {
redlock.releaseLock(lock, () => false);
return callback(err);
}
let uidValidity = mailbox.uidValidity;
let uid = message.uid;
server.notifier.addEntries(mailbox, false, {
command: 'EXISTS',
uid: message.uid,
message: message._id,
modseq: message.modseq
}, () => {
redlock.releaseLock(lock, () => {
server.notifier.fire(username, path);
return callback(null, true, {
uidValidity,
uid
});
});
});
});
});
});
});
});
};
module.exports = done => {
MongoClient.connect(config.mongo, (err, db) => {
if (err) {
@ -1289,6 +1168,7 @@ module.exports = done => {
let start = () => {
messageHandler = new MessageHandler(database);
server.indexer = new Indexer({
database

3
indexes.json
View file

@ -4,7 +4,8 @@
"name": "users",
"key": {
"username": 1
}
},
"unique": true
}]
}, {
"collection": "mailboxes",

8
imap-notifier.js → lib/imap-notifier.js
View file

@ -10,8 +10,6 @@ class ImapNotifier extends EventEmitter {
super();
this.database = options.database;
this.subsriber = redis.createClient();
this.publisher = redis.createClient();
let logfunc = (...args) => {
@ -27,6 +25,12 @@ class ImapNotifier extends EventEmitter {
error: logfunc.bind(null, 'ERROR')
};
if (options.pushOnly) {
// do not need to set up the following if we do not care about updates
return;
}
this.subsriber = redis.createClient();
this._listeners = new EventEmitter();
this._listeners.setMaxListeners(0);

167
lib/message-handler.js Normal file
View file

@ -0,0 +1,167 @@
'use strict';
const config = require('config');
const uuidV1 = require('uuid/v1');
const ObjectID = require('mongodb').ObjectID;
const RedFour = require('redfour');
const Indexer = require('../imap-core/lib/indexer/indexer');
const ImapNotifier = require('./imap-notifier');
class MessageHandler {
constructor(database) {
this.database = database;
this.indexer = new Indexer({
database
});
this.notifier = new ImapNotifier({
database,
pushOnly: true
});
this.redlock = new RedFour({
redis: config.redis,
namespace: 'wildduck'
});
}
getMailbox(options, callback) {
let query = {};
if (options.mailbox) {
if (typeof options.mailbox === 'object' && options.mailbox._id) {
return setImmediate(null, options.mailbox);
}
query._id = options.mailbox;
} else {
query.username = options.username;
query.path = options.path;
}
this.database.collection('mailboxes').findOne(query, (err, mailbox) => {
if (err) {
return callback(err);
}
if (!mailbox) {
let err = new Error('Mailbox is missing');
err.imapResponse = 'TRYCREATE';
return callback(err);
}
callback(null, mailbox);
});
}
add(options, callback) {
let id = new ObjectID();
let mimeTree = this.indexer.parseMimeTree(options.raw);
let size = this.indexer.getSize(mimeTree);
let bodystructure = this.indexer.getBodyStructure(mimeTree);
let envelope = this.indexer.getEnvelope(mimeTree);
let messageId = envelope[9] || ('<' + uuidV1() + '@wildduck.email>');
this.getMailbox(options, (err, mailbox) => {
if (err) {
return callback(err);
}
this.indexer.storeAttachments(id, mimeTree, 50 * 1024, err => {
if (err) {
return callback(err);
}
// Another server might be waiting for the lock like this.
this.redlock.waitAcquireLock(mailbox._id.toString(), 30 * 1000, 10 * 1000, (err, lock) => {
if (err) {
return callback(err);
}
if (!lock || !lock.success) {
// did not get a insert lock in 10 seconds
return callback(new Error('The user you are trying to contact is receiving mail at a rate that prevents additional messages from being delivered. Please resend your message at a later time'));
}
// acquire new UID+MODSEQ
this.database.collection('mailboxes').findOneAndUpdate({
_id: mailbox._id
}, {
$inc: {
// allocate bot UID and MODSEQ values so when journal is later sorted by
// modseq then UIDs are always in ascending order
uidNext: 1,
modifyIndex: 1
}
}, (err, item) => {
if (err) {
this.redlock.releaseLock(lock, () => false);
return callback(err);
}
if (!item || !item.value) {
// was not able to acquire a lock
let err = new Error('Mailbox is missing');
err.imapResponse = 'TRYCREATE';
this.redlock.releaseLock(lock, () => false);
return callback(err);
}
let mailbox = item.value;
let internaldate = options.date && new Date(options.date) || new Date();
let headerdate = mimeTree.parsedHeader.date && new Date(mimeTree.parsedHeader.date) || false;
if (!headerdate || headerdate.toString() === 'Invalid Date') {
headerdate = internaldate;
}
let message = {
_id: id,
mailbox: mailbox._id,
uid: mailbox.uidNext,
internaldate,
headerdate,
flags: [].concat(options.flags || []),
size,
meta: options.meta || {},
modseq: mailbox.modifyIndex + 1,
mimeTree,
envelope,
bodystructure,
messageId
};
this.database.collection('messages').insertOne(message, err => {
if (err) {
this.redlock.releaseLock(lock, () => false);
return callback(err);
}
let uidValidity = mailbox.uidValidity;
let uid = message.uid;
this.notifier.addEntries(mailbox, false, {
command: 'EXISTS',
uid: message.uid,
message: message._id,
modseq: message.modseq
}, () => {
this.redlock.releaseLock(lock, () => {
this.notifier.fire(mailbox.username, mailbox.path);
return callback(null, true, {
uidValidity,
uid
});
});
});
});
});
});
});
});
}
}
module.exports = MessageHandler;

30
lib/tools.js Normal file
View file

@ -0,0 +1,30 @@
'use strict';
const punycode = require('punycode');
function normalizeAddress(address, withNames) {
if (typeof address === 'string') {
address = {
address
};
}
if (!address || !address.address) {
return '';
}
let user = address.address.substr(0, address.address.lastIndexOf('@')).toLowerCase().trim();
let domain = address.address.substr(address.address.lastIndexOf('@') + 1).toLowerCase().trim();
let addr = user + '@' + punycode.toUnicode(domain);
if (withNames) {
return {
name: address.name || '',
address: addr
};
}
return addr;
}
module.exports = {
normalizeAddress
};

76
lmtp.js
View file

@ -6,9 +6,10 @@ const SMTPServer = require('smtp-server').SMTPServer;
const mongodb = require('mongodb');
const MongoClient = mongodb.MongoClient;
const crypto = require('crypto');
const punycode = require('punycode');
const tools = require('./lib/tools');
const MessageHandler = require('./lib/message-handler');
let imapServer;
let messageHandler;
let database;
const server = new SMTPServer({
@ -44,13 +45,18 @@ const server = new SMTPServer({
// Validate RCPT TO envelope address. Example allows all addresses that do not start with 'deny'
// If this method is not set, all addresses are allowed
onRcptTo(address, session, callback) {
let username = normalizeAddress(address.address);
let recipient = tools.normalizeAddress(address.address);
let username = recipient.replace(/\+[^@]*@/, '@');
if (session.users && session.users.has(username)) {
return callback();
}
database.collection('users').findOne({
username
}, (err, user) => {
if (err) {
log.error('LMTP', err);
log.error('SMTP', err);
return callback(new Error('Database error'));
}
if (!user) {
@ -58,10 +64,10 @@ const server = new SMTPServer({
}
if (!session.users) {
session.users = new Set();
session.users = new Map();
}
session.users.add(username);
session.users.set(username, recipient);
callback();
});
@ -105,23 +111,32 @@ const server = new SMTPServer({
return callback(null, 'Message queued as ' + hash.digest('hex').toUpperCase());
}
let username = users[stored++];
let username = users[stored][0];
let recipient = users[stored][1];
stored++;
// add Delivered-To
let header = Buffer.from('Delivered-To: ' + username + '\r\n');
chunks.unshift(header);
chunklen += header.length;
imapServer.addToMailbox(username, 'INBOX', {
source: 'LMTP',
from: normalizeAddress(session.envelope.mailFrom && session.envelope.mailFrom.address || ''),
to: session.envelope.rcptTo.map(item => normalizeAddress(item.address)),
origin: session.remoteAddress,
originhost: session.clientHostname,
transhost: session.hostNameAppearsAs,
transtype: session.transmissionType,
time: Date.now()
}, false, [], Buffer.concat(chunks, chunklen), err => {
messageHandler.add({
username,
path: 'INBOX',
meta: {
source: 'LMTP',
from: tools.normalizeAddress(session.envelope.mailFrom && session.envelope.mailFrom.address || ''),
to: recipient,
origin: session.remoteAddress,
originhost: session.clientHostname,
transhost: session.hostNameAppearsAs,
transtype: session.transmissionType,
time: Date.now()
},
date: false,
flags: false,
raw: Buffer.concat(chunks, chunklen)
}, err => {
// remove Delivered-To
chunks.shift();
chunklen -= header.length;
@ -139,30 +154,7 @@ const server = new SMTPServer({
}
});
function normalizeAddress(address, withNames) {
if (typeof address === 'string') {
address = {
address
};
}
if (!address || !address.address) {
return '';
}
let user = address.address.substr(0, address.address.lastIndexOf('@'));
let domain = address.address.substr(address.address.lastIndexOf('@') + 1);
let addr = user.trim() + '@' + punycode.toASCII(domain.toLowerCase().trim());
if (withNames) {
return {
name: address.name || '',
address: addr
};
}
return addr;
}
module.exports = (imap, done) => {
module.exports = done => {
if (!config.lmtp.enabled) {
return setImmediate(() => done(null, false));
}
@ -172,7 +164,7 @@ module.exports = (imap, done) => {
return;
}
database = mongo;
imapServer = imap;
messageHandler = new MessageHandler(database);
let started = false;

2
package.json
View file

@ -17,7 +17,7 @@
"grunt-eslint": "^19.0.0",
"grunt-mocha-test": "^0.13.2",
"mocha": "^3.2.0",
"nodemailer": "^3.1.7"
"nodemailer": "^3.1.8"
},
"dependencies": {
"addressparser": "^1.0.1",

76
smtp.js
View file

@ -6,9 +6,10 @@ const SMTPServer = require('smtp-server').SMTPServer;
const mongodb = require('mongodb');
const MongoClient = mongodb.MongoClient;
const crypto = require('crypto');
const punycode = require('punycode');
const tools = require('./lib/tools');
const MessageHandler = require('./lib/message-handler');
let imapServer;
let messageHandler;
let database;
const server = new SMTPServer({
@ -43,7 +44,12 @@ const server = new SMTPServer({
// Validate RCPT TO envelope address. Example allows all addresses that do not start with 'deny'
// If this method is not set, all addresses are allowed
onRcptTo(address, session, callback) {
let username = normalizeAddress(address.address);
let recipient = tools.normalizeAddress(address.address);
let username = recipient.replace(/\+[^@]*@/, '@');
if (session.users && session.users.has(username)) {
return callback();
}
database.collection('users').findOne({
username
@ -57,10 +63,10 @@ const server = new SMTPServer({
}
if (!session.users) {
session.users = new Set();
session.users = new Map();
}
session.users.add(username);
session.users.set(username, recipient);
callback();
});
@ -104,29 +110,38 @@ const server = new SMTPServer({
return callback(null, 'Message queued as ' + hash.digest('hex').toUpperCase());
}
let username = users[stored++];
let username = users[stored][0];
let recipient = users[stored][1];
stored++;
// add Delivered-To
let header = Buffer.from('Delivered-To: ' + username + '\r\n');
chunks.unshift(header);
chunklen += header.length;
imapServer.addToMailbox(username, 'INBOX', {
source: 'SMTP',
from: normalizeAddress(session.envelope.mailFrom && session.envelope.mailFrom.address || ''),
to: session.envelope.rcptTo.map(item => normalizeAddress(item.address)),
origin: session.remoteAddress,
originhost: session.clientHostname,
transhost: session.hostNameAppearsAs,
transtype: session.transmissionType,
time: Date.now()
}, false, [], Buffer.concat(chunks, chunklen), err => {
messageHandler.add({
username,
path: 'INBOX',
meta: {
source: 'SMTP',
from: tools.normalizeAddress(session.envelope.mailFrom && session.envelope.mailFrom.address || ''),
to: recipient,
origin: session.remoteAddress,
originhost: session.clientHostname,
transhost: session.hostNameAppearsAs,
transtype: session.transmissionType,
time: Date.now()
},
date: false,
flags: false,
raw: Buffer.concat(chunks, chunklen)
}, err => {
// remove Delivered-To
chunks.shift();
chunklen -= header.length;
if (err) {
return callback(err);
log.error('LMTP', err);
}
storeNext();
@ -138,30 +153,7 @@ const server = new SMTPServer({
}
});
function normalizeAddress(address, withNames) {
if (typeof address === 'string') {
address = {
address
};
}
if (!address || !address.address) {
return '';
}
let user = address.address.substr(0, address.address.lastIndexOf('@'));
let domain = address.address.substr(address.address.lastIndexOf('@') + 1);
let addr = user.trim() + '@' + punycode.toASCII(domain.toLowerCase().trim());
if (withNames) {
return {
name: address.name || '',
address: addr
};
}
return addr;
}
module.exports = (imap, done) => {
module.exports = done => {
if (!config.smtp.enabled) {
return setImmediate(() => done(null, false));
}
@ -171,7 +163,7 @@ module.exports = (imap, done) => {
return;
}
database = mongo;
imapServer = imap;
messageHandler = new MessageHandler(database);
let started = false;

8
worker.js
View file

@ -7,22 +7,22 @@ let lmtp = require('./lmtp');
let smtp = require('./smtp');
let api = require('./api');
imap((err, imap) => {
imap(err => {
if (err) {
log.error('App', 'Failed to start IMAP server');
return process.exit(1);
}
lmtp(imap, err => {
lmtp(err => {
if (err) {
log.error('App', 'Failed to start LMTP server');
return process.exit(1);
}
smtp(imap, err => {
smtp(err => {
if (err) {
log.error('App', 'Failed to start SMTP server');
return process.exit(1);
}
api(imap, err => {
api(err => {
if (err) {
log.error('App', 'Failed to start API server');
return process.exit(1);

1954
yarn.lock
View file

File diff suppressed because it is too large Load diff