This commit is contained in:
Andris Reinman 2018-10-23 11:38:08 +03:00
parent 56fe162d1c
commit 077ff4ddb5
4 changed files with 240 additions and 468 deletions

View file

@ -477,10 +477,7 @@ module.exports = (db, server, messageHandler, userHandler) => {
meta,
date: false,
flags: ['\\Seen'].concat(options.isDraft ? '\\Draft' : []),
// always insert messages
skipExisting: false
flags: ['\\Seen'].concat(options.isDraft ? '\\Draft' : [])
};
if (raw) {

View file

@ -533,10 +533,7 @@ class FilterHandler {
filters: matchingFilters,
date: false,
flags,
// if similar message exists, then skip
skipExisting: true
flags
};
if (outbound && outbound.length) {

View file

@ -124,501 +124,279 @@ class MessageHandler {
return callback(err);
}
this.checkExistingMessage(
mailboxData,
{
id,
let cleanup = (...args) => {
if (!args[0]) {
return callback(...args);
}
let attachmentIds = Object.keys(mimeTree.attachmentMap || {}).map(key => mimeTree.attachmentMap[key]);
if (!attachmentIds.length) {
return callback(...args);
}
this.attachmentStorage.deleteMany(attachmentIds, maildata.magic, () => callback(...args));
};
this.indexer.storeNodeBodies(maildata, mimeTree, err => {
if (err) {
return cleanup(err);
}
// prepare message object
let messageData = {
_id: id,
// should be kept when COPY'ing or MOVE'ing
root: id,
v: consts.SCHEMA_VERSION,
// if true then expires after rdate + retention
exp: !!mailboxData.retention,
rdate: Date.now() + (mailboxData.retention || 0),
// make sure the field exists. it is set to true when user is deleted
userDeleted: false,
idate,
hdate,
msgid,
flags,
size
},
options,
(...args) => {
if (args[0] || args[1]) {
return callback(...args);
size,
// some custom metadata about the delivery
meta: options.meta || {},
// list filter IDs that matched this message
filters: Array.isArray(options.filters) ? options.filters : [].concat(options.filters || []),
headers,
mimeTree,
envelope,
bodystructure,
msgid,
// use boolean for more commonly used (and searched for) flags
unseen: !flags.includes('\\Seen'),
flagged: flags.includes('\\Flagged'),
undeleted: !flags.includes('\\Deleted'),
draft: flags.includes('\\Draft'),
magic: maildata.magic,
subject
};
if (options.outbound) {
messageData.outbound = [].concat(options.outbound || []);
}
if (options.forwardTargets) {
messageData.forwardTargets = [].concat(options.forwardTargets || []);
}
if (maildata.attachments && maildata.attachments.length) {
messageData.attachments = maildata.attachments;
messageData.ha = !!maildata.attachments.find(a => !a.related);
} else {
messageData.ha = false;
}
if (maildata.text) {
messageData.text = maildata.text.replace(/\r\n/g, '\n').trim();
// text is indexed with a fulltext index, so only store the beginning of it
if (messageData.text.length > consts.MAX_PLAINTEXT_INDEXED) {
messageData.textFooter = messageData.text.substr(consts.MAX_PLAINTEXT_INDEXED);
messageData.text = messageData.text.substr(0, consts.MAX_PLAINTEXT_INDEXED);
// truncate remaining text if total length exceeds maximum allowed
if (
consts.MAX_PLAINTEXT_CONTENT > consts.MAX_PLAINTEXT_INDEXED &&
messageData.textFooter.length > consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED
) {
messageData.textFooter = messageData.textFooter.substr(0, consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED);
}
}
messageData.text =
messageData.text.length <= consts.MAX_PLAINTEXT_CONTENT
? messageData.text
: messageData.text.substr(0, consts.MAX_PLAINTEXT_CONTENT);
let cleanup = (...args) => {
if (!args[0]) {
return callback(...args);
messageData.intro = messageData.text
// assume we get the intro text from first 2 kB
.substr(0, 2 * 1024)
// remove quoted parts
// "> quote from previous message"
.replace(/^>.*$/gm, '')
// remove lines with repetetive chars
// "---------------------"
.replace(/^\s*(.)\1+\s*$/gm, '')
// join lines
.replace(/\s+/g, ' ')
.trim();
if (messageData.intro.length > 128) {
let intro = messageData.intro.substr(0, 128);
let lastSp = intro.lastIndexOf(' ');
if (lastSp > 0) {
intro = intro.substr(0, lastSp);
}
messageData.intro = intro + '…';
}
}
let attachmentIds = Object.keys(mimeTree.attachmentMap || {}).map(key => mimeTree.attachmentMap[key]);
if (!attachmentIds.length) {
return callback(...args);
if (maildata.html && maildata.html.length) {
let htmlSize = 0;
messageData.html = maildata.html
.map(html => {
if (htmlSize >= consts.MAX_HTML_CONTENT || !html) {
return '';
}
if (htmlSize + Buffer.byteLength(html) <= consts.MAX_HTML_CONTENT) {
htmlSize += Buffer.byteLength(html);
return html;
}
html = html.substr(0, htmlSize + Buffer.byteLength(html) - consts.MAX_HTML_CONTENT);
htmlSize += Buffer.byteLength(html);
return html;
})
.filter(html => html);
}
this.users.collection('users').updateOne(
{
_id: mailboxData.user
},
{
$inc: {
storageUsed: size
}
this.attachmentStorage.deleteMany(attachmentIds, maildata.magic, () => callback(...args));
};
this.indexer.storeNodeBodies(maildata, mimeTree, err => {
},
err => {
if (err) {
return cleanup(err);
}
// prepare message object
let messageData = {
_id: id,
// should be kept when COPY'ing or MOVE'ing
root: id,
v: consts.SCHEMA_VERSION,
// if true then expires after rdate + retention
exp: !!mailboxData.retention,
rdate: Date.now() + (mailboxData.retention || 0),
// make sure the field exists. it is set to true when user is deleted
userDeleted: false,
idate,
hdate,
flags,
size,
// some custom metadata about the delivery
meta: options.meta || {},
// list filter IDs that matched this message
filters: Array.isArray(options.filters) ? options.filters : [].concat(options.filters || []),
headers,
mimeTree,
envelope,
bodystructure,
msgid,
// use boolean for more commonly used (and searched for) flags
unseen: !flags.includes('\\Seen'),
flagged: flags.includes('\\Flagged'),
undeleted: !flags.includes('\\Deleted'),
draft: flags.includes('\\Draft'),
magic: maildata.magic,
subject
let rollback = err => {
this.users.collection('users').updateOne(
{
_id: mailboxData.user
},
{
$inc: {
storageUsed: -size
}
},
() => {
cleanup(err);
}
);
};
if (options.outbound) {
messageData.outbound = [].concat(options.outbound || []);
}
if (options.forwardTargets) {
messageData.forwardTargets = [].concat(options.forwardTargets || []);
}
if (maildata.attachments && maildata.attachments.length) {
messageData.attachments = maildata.attachments;
messageData.ha = !!maildata.attachments.find(a => !a.related);
} else {
messageData.ha = false;
}
if (maildata.text) {
messageData.text = maildata.text.replace(/\r\n/g, '\n').trim();
// text is indexed with a fulltext index, so only store the beginning of it
if (messageData.text.length > consts.MAX_PLAINTEXT_INDEXED) {
messageData.textFooter = messageData.text.substr(consts.MAX_PLAINTEXT_INDEXED);
messageData.text = messageData.text.substr(0, consts.MAX_PLAINTEXT_INDEXED);
// truncate remaining text if total length exceeds maximum allowed
if (
consts.MAX_PLAINTEXT_CONTENT > consts.MAX_PLAINTEXT_INDEXED &&
messageData.textFooter.length > consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED
) {
messageData.textFooter = messageData.textFooter.substr(0, consts.MAX_PLAINTEXT_CONTENT - consts.MAX_PLAINTEXT_INDEXED);
}
}
messageData.text =
messageData.text.length <= consts.MAX_PLAINTEXT_CONTENT
? messageData.text
: messageData.text.substr(0, consts.MAX_PLAINTEXT_CONTENT);
messageData.intro = messageData.text
// assume we get the intro text from first 2 kB
.substr(0, 2 * 1024)
// remove quoted parts
// "> quote from previous message"
.replace(/^>.*$/gm, '')
// remove lines with repetetive chars
// "---------------------"
.replace(/^\s*(.)\1+\s*$/gm, '')
// join lines
.replace(/\s+/g, ' ')
.trim();
if (messageData.intro.length > 128) {
let intro = messageData.intro.substr(0, 128);
let lastSp = intro.lastIndexOf(' ');
if (lastSp > 0) {
intro = intro.substr(0, lastSp);
}
messageData.intro = intro + '…';
}
}
if (maildata.html && maildata.html.length) {
let htmlSize = 0;
messageData.html = maildata.html
.map(html => {
if (htmlSize >= consts.MAX_HTML_CONTENT || !html) {
return '';
}
if (htmlSize + Buffer.byteLength(html) <= consts.MAX_HTML_CONTENT) {
htmlSize += Buffer.byteLength(html);
return html;
}
html = html.substr(0, htmlSize + Buffer.byteLength(html) - consts.MAX_HTML_CONTENT);
htmlSize += Buffer.byteLength(html);
return html;
})
.filter(html => html);
}
this.users.collection('users').updateOne(
// acquire new UID+MODSEQ
this.database.collection('mailboxes').findOneAndUpdate(
{
_id: mailboxData.user
_id: mailboxData._id
},
{
$inc: {
storageUsed: size
// allocate bot UID and MODSEQ values so when journal is later sorted by
// modseq then UIDs are always in ascending order
uidNext: 1,
modifyIndex: 1
}
},
err => {
{
// use original value to get correct UIDNext
returnOriginal: true
},
(err, item) => {
if (err) {
return cleanup(err);
return rollback(err);
}
let rollback = err => {
this.users.collection('users').updateOne(
{
_id: mailboxData.user
},
{
$inc: {
storageUsed: -size
}
},
() => {
cleanup(err);
}
);
};
if (!item || !item.value) {
// was not able to acquire a lock
let err = new Error('Mailbox is missing');
err.imapResponse = 'TRYCREATE';
return rollback(err);
}
// acquire new UID+MODSEQ
this.database.collection('mailboxes').findOneAndUpdate(
{
_id: mailboxData._id
},
{
$inc: {
// allocate bot UID and MODSEQ values so when journal is later sorted by
// modseq then UIDs are always in ascending order
uidNext: 1,
modifyIndex: 1
}
},
{
// use original value to get correct UIDNext
returnOriginal: true
},
(err, item) => {
let mailboxData = item.value;
// updated message object by setting mailbox specific values
messageData.mailbox = mailboxData._id;
messageData.user = mailboxData.user;
messageData.uid = mailboxData.uidNext;
messageData.modseq = mailboxData.modifyIndex + 1;
if (!['\\Junk', '\\Trash'].includes(mailboxData.specialUse) && !flags.includes('\\Deleted')) {
messageData.searchable = true;
}
if (mailboxData.specialUse === '\\Junk') {
messageData.junk = true;
}
this.getThreadId(mailboxData.user, subject, mimeTree, (err, thread) => {
if (err) {
return rollback(err);
}
messageData.thread = thread;
this.database.collection('messages').insertOne(messageData, err => {
if (err) {
return rollback(err);
}
if (!item || !item.value) {
// was not able to acquire a lock
let err = new Error('Mailbox is missing');
err.imapResponse = 'TRYCREATE';
return rollback(err);
let logTime = messageData.meta.time || new Date();
if (typeof logTime === 'number') {
logTime = new Date(logTime);
}
let mailboxData = item.value;
let uidValidity = mailboxData.uidValidity;
let uid = messageData.uid;
// updated message object by setting mailbox specific values
messageData.mailbox = mailboxData._id;
messageData.user = mailboxData.user;
messageData.uid = mailboxData.uidNext;
messageData.modseq = mailboxData.modifyIndex + 1;
if (!['\\Junk', '\\Trash'].includes(mailboxData.specialUse) && !flags.includes('\\Deleted')) {
messageData.searchable = true;
if (
options.session &&
options.session.selected &&
options.session.selected.mailbox &&
options.session.selected.mailbox.toString() === mailboxData._id.toString()
) {
options.session.writeStream.write(options.session.formatResponse('EXISTS', messageData.uid));
}
if (mailboxData.specialUse === '\\Junk') {
messageData.junk = true;
}
this.getThreadId(mailboxData.user, subject, mimeTree, (err, thread) => {
if (err) {
return rollback(err);
this.notifier.addEntries(
mailboxData,
{
command: 'EXISTS',
uid: messageData.uid,
ignore: options.session && options.session.id,
message: messageData._id,
modseq: messageData.modseq,
unseen: messageData.unseen
},
() => {
this.notifier.fire(mailboxData.user);
return cleanup(null, true, {
uidValidity,
uid,
id: messageData._id,
mailbox: mailboxData._id,
status: 'new'
});
}
messageData.thread = thread;
this.database.collection('messages').insertOne(messageData, err => {
if (err) {
return rollback(err);
}
let logTime = messageData.meta.time || new Date();
if (typeof logTime === 'number') {
logTime = new Date(logTime);
}
let uidValidity = mailboxData.uidValidity;
let uid = messageData.uid;
if (
options.session &&
options.session.selected &&
options.session.selected.mailbox &&
options.session.selected.mailbox.toString() === mailboxData._id.toString()
) {
options.session.writeStream.write(options.session.formatResponse('EXISTS', messageData.uid));
}
this.notifier.addEntries(
mailboxData,
{
command: 'EXISTS',
uid: messageData.uid,
ignore: options.session && options.session.id,
message: messageData._id,
modseq: messageData.modseq,
unseen: messageData.unseen
},
() => {
this.notifier.fire(mailboxData.user);
return cleanup(null, true, {
uidValidity,
uid,
id: messageData._id,
mailbox: mailboxData._id,
status: 'new'
});
}
);
});
});
}
);
);
});
});
}
);
});
}
);
});
});
}
checkExistingMessage(mailboxData, messageOpts, options, callback) {
// if a similar message already exists then update existing one
let queryOpts = {};
if (options.skipExisting) {
// no need to load extra data when we only need to know the basics
queryOpts.projection = {
_id: true,
uid: true,
outbound: true,
mailbox: true,
size: true
};
}
let query = {
mailbox: mailboxData._id,
hdate: messageOpts.hdate,
msgid: messageOpts.msgid,
uid: {
$gt: 0,
$lt: mailboxData.uidNext
}
};
this.database.collection('messages').findOne(query, queryOpts, (err, messageData) => {
if (err) {
return callback(err);
}
if (!messageData) {
// nothing to do here, continue adding message
return callback();
}
if (Math.abs(messageData.size - messageOpts.size) > 128) {
// too big difference in size, add as new
return callback();
}
let existingId = messageData._id;
let existingUid = messageData.uid;
let existingMailbox = messageData.mailbox;
let outbound = [].concat(messageData.outbound || []).concat(options.outbound || []);
if (outbound) {
messageData.outbound = outbound;
}
if (options.skipExisting) {
// message already exists, just skip it
if (options.outbound) {
// new outbound ID's. update
return this.database.collection('messages').findOneAndUpdate(
{
_id: messageData._id,
mailbox: messageData.mailbox,
uid: messageData.uid
},
{
$addToSet: {
outbound: { $each: [].concat(options.outbound || []) }
}
},
{
returnOriginal: true,
projection: {
_id: true,
outbound: true
}
},
() =>
callback(null, true, {
uid: existingUid,
id: existingId,
mailbox: mailboxData._id,
status: 'skip'
})
);
}
return callback(null, true, {
uid: existingUid,
id: existingId,
mailbox: mailboxData._id,
status: 'skip'
});
}
// As duplicate message was found, update UID, MODSEQ and FLAGS
// acquire new UID+MODSEQ
this.database.collection('mailboxes').findOneAndUpdate(
{
_id: mailboxData._id
},
{
$inc: {
// allocate both UID and MODSEQ values so when journal is later sorted by
// modseq then UIDs are always in ascending order
uidNext: 1,
modifyIndex: 1
}
},
{
returnOriginal: true
},
(err, item) => {
if (err) {
return callback(err);
}
if (!item || !item.value) {
// was not able to acquire a lock
let err = new Error('Mailbox is missing');
err.imapResponse = 'TRYCREATE';
return callback(err);
}
let mailboxData = item.value;
let newUid = mailboxData.uidNext;
let newModseq = mailboxData.modifyIndex + 1;
// UID is immutable, so if we want to change it, we need to copy the message
messageData._id = messageOpts.id;
// inserted message might not be in the same mailbox as the deleted one
messageData.mailbox = mailboxData._id;
messageData.uid = newUid;
messageData.modseq = newModseq;
messageData.flags = messageOpts.flags;
messageData.unseen = !messageOpts.flags.includes('\\Seen');
messageData.flagged = messageOpts.flags.includes('\\Flagged');
messageData.undeleted = !messageOpts.flags.includes('\\Deleted');
messageData.draft = messageOpts.flags.includes('\\Draft');
this.database.collection('messages').insertOne(messageData, err => {
if (err) {
return callback(err);
}
this.database.collection('messages').deleteOne(
{
_id: existingId,
// hash key
mailbox: existingMailbox,
uid: existingUid
},
err => {
if (err) {
// TODO: how to resolve this? we might end up with two copies of the same message :S
return callback(err);
}
if (options.session && options.session.selected && options.session.selected.mailbox.toString() === existingMailbox.toString()) {
options.session.writeStream.write(options.session.formatResponse('EXPUNGE', existingUid));
}
if (options.session && options.session.selected && options.session.selected.mailbox.toString() === mailboxData._id.toString()) {
options.session.writeStream.write(options.session.formatResponse('EXISTS', messageData.uid));
}
this.notifier.addEntries(
existingMailbox.toString() === mailboxData._id.toString() ? mailboxData : existingMailbox,
{
command: 'EXPUNGE',
ignore: options.session && options.session.id,
uid: existingUid,
message: existingId,
unseen: messageData.unseen,
// modseq is needed to avoid updating mailbox entry
modseq: newModseq
},
() => {
this.notifier.addEntries(
mailboxData,
{
command: 'EXISTS',
uid: messageData.uid,
ignore: options.session && options.session.id,
message: messageData._id,
modseq: messageData.modseq,
unseen: messageData.unseen
},
() => {
this.notifier.fire(mailboxData.user);
return callback(null, true, {
uidValidity: mailboxData.uidValidity,
uid: newUid,
id: messageData._id,
mailbox: mailboxData._id,
status: 'update'
});
}
);
}
);
}
);
});
}
);
);
});
});
});
}

View file

@ -1,6 +1,6 @@
{
"name": "wildduck",
"version": "1.9.2",
"version": "1.10.0",
"description": "IMAP/POP3 server built with Node.js and MongoDB",
"main": "server.js",
"scripts": {