From 4138bf2a2ff867b903139aea8125cdb4fa1583de Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Sun, 6 Aug 2017 21:25:10 +0300 Subject: [PATCH] refactor attachment storage --- api.js | 3 + imap-core/lib/indexer/indexer.js | 137 +++------------------- imap.js | 29 +++-- lib/api/messages.js | 26 ++--- lib/attachment-storage.js | 193 +++++++++++++++++++++++++++++++ lib/handlers/on-copy.js | 23 +--- lib/handlers/on-expunge.js | 25 +--- lib/handlers/on-fetch.js | 4 +- lib/message-handler.js | 106 ++++++++--------- lmtp.js | 7 +- pop3.js | 13 ++- 11 files changed, 318 insertions(+), 248 deletions(-) create mode 100644 lib/attachment-storage.js diff --git a/api.js b/api.js index 1d39e4e9..cf6a5bad 100644 --- a/api.js +++ b/api.js @@ -100,18 +100,21 @@ module.exports = done => { database: db.database, redis: db.redis }); + messageHandler = new MessageHandler({ database: db.database, users: db.users, gridfs: db.gridfs, redis: db.redis }); + userHandler = new UserHandler({ database: db.database, users: db.users, redis: db.redis, messageHandler }); + mailboxHandler = new MailboxHandler({ database: db.database, users: db.users, diff --git a/imap-core/lib/indexer/indexer.js b/imap-core/lib/indexer/indexer.js index 2b0c4d39..f05b47da 100644 --- a/imap-core/lib/indexer/indexer.js +++ b/imap-core/lib/indexer/indexer.js @@ -6,8 +6,6 @@ const PassThrough = stream.PassThrough; const BodyStructure = require('./body-structure'); const createEnvelope = require('./create-envelope'); const parseMimeTree = require('./parse-mime-tree'); -const ObjectID = require('mongodb').ObjectID; -const GridFSBucket = require('mongodb').GridFSBucket; const libmime = require('libmime'); const libqp = require('libqp'); const libbase64 = require('libbase64'); @@ -16,25 +14,12 @@ const he = require('he'); const htmlToText = require('html-to-text'); const crypto = require('crypto'); -let cryptoAsync; -try { - cryptoAsync = require('@ronomon/crypto-async'); // eslint-disable-line global-require -} catch (E) { - // ignore -} - class Indexer { constructor(options) { this.options = options || {}; this.fetchOptions = this.options.fetchOptions || {}; - this.database = this.options.database; - this.gridfs = this.options.gridfs || this.options.database; - if (this.gridfs) { - this.gridstore = new GridFSBucket(this.gridfs, { - bucketName: 'attachments' - }); - } + this.attachmentStorage = this.options.attachmentStorage; // create logger this.logger = this.options.logger || { @@ -195,7 +180,11 @@ class Indexer { } else if (node.attachmentId && !skipExternal) { append(false, true); // force newline between header and contents - let attachmentStream = this.gridstore.openDownloadStream(node.attachmentId); + let attachmentId = node.attachmentId; + if (mimeTree.attachmentMap && mimeTree.attachmentMap[node.attachmentId]) { + attachmentId = mimeTree.attachmentMap[node.attachmentId]; + } + let attachmentStream = this.attachmentStorage.createReadStream(attachmentId); attachmentStream.once('error', err => { res.emit('error', err); @@ -267,16 +256,13 @@ class Indexer { */ getMaildata(messageId, mimeTree) { let magic = parseInt(crypto.randomBytes(2).toString('hex'), 16); - let map = {}; let maildata = { nodes: [], attachments: [], text: '', html: [], // magic number to append to increment stored attachment object counter - magic, - // match ids referenced in document to actual attachment ids - map + magic }; let idcount = 0; @@ -363,7 +349,6 @@ class Indexer { // remove attachments and very large text nodes from the mime tree if (!isMultipart && node.body && node.body.length && (!isInlineText || node.size > 300 * 1024)) { let attachmentId = 'ATT' + leftPad(++idcount, '0', 5); - map[attachmentId] = new ObjectID(); let fileName = (node.parsedHeader['content-disposition'] && @@ -371,6 +356,7 @@ class Indexer { node.parsedHeader['content-disposition'].params.filename) || (node.parsedHeader['content-type'] && node.parsedHeader['content-type'].params && node.parsedHeader['content-type'].params.name) || false; + let contentId = (node.parsedHeader['content-id'] || '').toString().replace(/<|>/g, '').trim(); if (fileName) { @@ -391,21 +377,9 @@ class Indexer { // push to queue maildata.nodes.push({ attachmentId, - options: { - fsync: true, - contentType, - // metadata should include only minimally required information, this would allow - // to share attachments between different messages if the content is exactly the same - // even though metadata (filename, content-disposition etc) might not - metadata: { - // values to detect if there are messages that reference to this attachment or not - m: maildata.magic, - c: 1, - - // how to decode contents if a webclient or API asks for the attachment - transferEncoding - } - }, + magic: maildata.magic, + contentType, + transferEncoding, body: node.body }); @@ -463,82 +437,19 @@ class Indexer { storeNodeBodies(messageId, maildata, mimeTree, callback) { let pos = 0; let nodes = maildata.nodes; + mimeTree.attachmentMap = {}; let storeNode = () => { if (pos >= nodes.length) { - // replace attachment IDs with ObjectIDs in the mimeTree - let walk = (node, next) => { - if (node.attachmentId && maildata.map[node.attachmentId]) { - node.attachmentId = maildata.map[node.attachmentId]; - } - - if (Array.isArray(node.childNodes)) { - let pos = 0; - let processChildNodes = () => { - if (pos >= node.childNodes.length) { - return next(); - } - let childNode = node.childNodes[pos++]; - walk(childNode, () => processChildNodes()); - }; - processChildNodes(); - } else { - next(); - } - }; - - return walk(mimeTree, () => callback(null, true)); + return callback(null, true); } let node = nodes[pos++]; - - calculateHash(node.body, (err, hash) => { + this.attachmentStorage.create(node, (err, id) => { if (err) { return callback(err); } - - this.gridfs.collection('attachments.files').findOneAndUpdate({ - 'metadata.h': hash - }, { - $inc: { - 'metadata.c': 1, - 'metadata.m': maildata.magic - } - }, { - returnOriginal: false - }, (err, result) => { - if (err) { - return callback(err); - } - - if (result && result.value) { - maildata.map[node.attachmentId] = result.value._id; - return storeNode(); - } - - let returned = false; - - node.options.metadata.h = hash; - - let store = this.gridstore.openUploadStreamWithId(maildata.map[node.attachmentId], null, node.options); - - store.once('error', err => { - if (returned) { - return; - } - returned = true; - callback(err); - }); - - store.once('finish', () => { - if (returned) { - return; - } - returned = true; - return storeNode(); - }); - - store.end(node.body); - }); + mimeTree.attachmentMap[node.attachmentId] = id; + return storeNode(); }); }; @@ -800,20 +711,4 @@ function leftPad(val, chr, len) { return chr.repeat(len - val.toString().length) + val; } -function calculateHash(input, callback) { - let algo = 'sha256'; - - if (!cryptoAsync) { - setImmediate(() => callback(null, crypto.createHash(algo).update(input).digest('hex'))); - return; - } - - cryptoAsync.hash(algo, input, (err, hash) => { - if (err) { - return callback(err); - } - return callback(null, hash.toString('hex')); - }); -} - module.exports = Indexer; diff --git a/imap.js b/imap.js index 493a5f3b..49286418 100644 --- a/imap.js +++ b/imap.js @@ -199,7 +199,7 @@ function clearExpiredMessages() { mailbox: true, uid: true, size: true, - map: true, + 'mimeTree.attachmentMap': true, magic: true, unseen: true }); @@ -295,9 +295,24 @@ module.exports = done => { redis: db.redis }); - messageHandler = new MessageHandler({ database: db.database, gridfs: db.gridfs, redis: db.redis }); - userHandler = new UserHandler({ database: db.database, users: db.users, redis: db.redis }); - mailboxHandler = new MailboxHandler({ database: db.database, users: db.users, redis: db.redis, notifier: server.notifier }); + messageHandler = new MessageHandler({ + database: db.database, + gridfs: db.gridfs, + redis: db.redis + }); + + userHandler = new UserHandler({ + database: db.database, + users: db.users, + redis: db.redis + }); + + mailboxHandler = new MailboxHandler({ + database: db.database, + users: db.users, + redis: db.redis, + notifier: server.notifier + }); let started = false; @@ -324,7 +339,7 @@ module.exports = done => { }); // setup command handlers for the server instance - server.onFetch = onFetch(server); + server.onFetch = onFetch(server, messageHandler); server.onAuth = onAuth(server, userHandler); server.onList = onList(server); server.onLsub = onLsub(server); @@ -337,8 +352,8 @@ module.exports = done => { server.onStatus = onStatus(server); server.onAppend = onAppend(server, messageHandler); server.onStore = onStore(server); - server.onExpunge = onExpunge(server); - server.onCopy = onCopy(server); + server.onExpunge = onExpunge(server, messageHandler); + server.onCopy = onCopy(server, messageHandler); server.onMove = onMove(server, messageHandler); server.onSearch = onSearch(server); server.onGetQuotaRoot = onGetQuotaRoot(server); diff --git a/lib/api/messages.js b/lib/api/messages.js index 860dd2ec..e1c830a3 100644 --- a/lib/api/messages.js +++ b/lib/api/messages.js @@ -5,7 +5,6 @@ const MongoPaging = require('mongo-cursor-pagination'); const addressparser = require('addressparser'); const ObjectID = require('mongodb').ObjectID; const tools = require('../tools'); -const GridFSBucket = require('mongodb').GridFSBucket; const libbase64 = require('libbase64'); const libqp = require('libqp'); @@ -667,7 +666,7 @@ module.exports = (db, server, messageHandler) => { _id: true, user: true, attachments: true, - map: true + 'mimeTree.attachmentMap': true } }, (err, messageData) => { if (err) { @@ -683,7 +682,7 @@ module.exports = (db, server, messageHandler) => { return next(); } - let attachmentId = messageData.map[attachment]; + let attachmentId = messageData.mimeTree.attachmentMap && messageData.mimeTree.attachmentMap[attachment]; if (!attachmentId) { res.json({ error: 'This attachment does not exist' @@ -691,36 +690,25 @@ module.exports = (db, server, messageHandler) => { return next(); } - db.gridfs.collection('attachments.files').findOne({ - _id: attachmentId - }, (err, attachmentData) => { + messageHandler.attachmentStorage.get(attachmentId, (err, attachmentData) => { if (err) { res.json({ error: err.message }); return next(); } - if (!attachmentData) { - res.json({ - error: 'This attachment does not exist' - }); - return next(); - } res.writeHead(200, { 'Content-Type': attachmentData.contentType || 'application/octet-stream' }); - let bucket = new GridFSBucket(db.gridfs, { - bucketName: 'attachments' - }); - let attachmentStream = bucket.openDownloadStream(attachmentId); + let attachmentStream = messageHandler.attachmentStorage.createReadStream(attachmentId); attachmentStream.once('error', err => res.emit('error', err)); - if (attachmentData.metadata.transferEncoding === 'base64') { + if (attachmentData.transferEncoding === 'base64') { attachmentStream.pipe(new libbase64.Decoder()).pipe(res); - } else if (attachmentData.metadata.transferEncoding === 'quoted-printable') { + } else if (attachmentData.transferEncoding === 'quoted-printable') { attachmentStream.pipe(new libqp.Decoder()).pipe(res); } else { attachmentStream.pipe(res); @@ -876,7 +864,7 @@ module.exports = (db, server, messageHandler) => { mailbox: true, uid: true, size: true, - map: true, + 'mimeTree.attachmentMap': true, magic: true, unseen: true } diff --git a/lib/attachment-storage.js b/lib/attachment-storage.js new file mode 100644 index 00000000..0817c72b --- /dev/null +++ b/lib/attachment-storage.js @@ -0,0 +1,193 @@ +'use strict'; + +const ObjectID = require('mongodb').ObjectID; +const crypto = require('crypto'); +const GridFSBucket = require('mongodb').GridFSBucket; +let cryptoAsync; +try { + cryptoAsync = require('@ronomon/crypto-async'); // eslint-disable-line global-require +} catch (E) { + // ignore +} + +class AttachmentStorage { + constructor(options) { + this.bucketName = options.bucket || 'attachments'; + this.gridfs = options.gridfs; + this.gridstore = new GridFSBucket(this.gridfs, { + bucketName: this.bucketName + }); + } + + get(attachmentId, callback) { + this.gridfs.collection('attachments.files').findOne({ + _id: attachmentId + }, (err, attachmentData) => { + if (err) { + return callback(err); + } + if (!attachmentData) { + return callback(new Error('This attachment does not exist')); + } + + return callback(null, { + contentType: attachmentData.contentType, + transferEncoding: attachmentData.metadata.transferEncoding, + metadata: attachmentData.metadata + }); + }); + } + + create(attachment, callback) { + this.calculateHash(attachment.body, (err, hash) => { + if (err) { + return callback(err); + } + + this.gridfs.collection(this.bucketName + '.files').findOneAndUpdate({ + 'metadata.h': hash + }, { + $inc: { + 'metadata.c': 1, + 'metadata.m': attachment.magic + } + }, { + returnOriginal: false + }, (err, result) => { + if (err) { + return callback(err); + } + + if (result && result.value) { + return callback(null, result.value._id); + } + + let returned = false; + + let id = new ObjectID(); + let metadata = { + h: hash, + m: attachment.magic, + c: 1, + transferEncoding: attachment.transferEncoding + }; + Object.keys(attachment.metadata || {}).forEach(key => { + if (!(key in attachment.metadata)) { + metadata[key] = attachment.metadata[key]; + } + }); + + let store = this.gridstore.openUploadStreamWithId(id, null, { + contentType: attachment.contentType, + metadata + }); + + store.once('error', err => { + if (returned) { + return; + } + returned = true; + callback(err); + }); + + store.once('finish', () => { + if (returned) { + return; + } + returned = true; + return callback(null, id); + }); + + store.end(attachment.body); + }); + }); + } + + createReadStream(id) { + return this.gridstore.openDownloadStream(id); + } + + deleteMany(ids, magic, callback) { + let pos = 0; + let deleteNext = () => { + if (pos >= ids.length) { + return callback(null, true); + } + let id = ids[pos++]; + this.delete(id, magic, deleteNext); + }; + deleteNext(); + } + + updateMany(ids, count, magic, callback) { + // update attachments + this.gridfs.collection(this.bucketName + '.files').updateMany( + { + _id: { + $in: ids + } + }, + { + $inc: { + 'metadata.c': count, + 'metadata.m': magic + } + }, + { + multi: true, + w: 1 + }, + callback + ); + } + + delete(id, magic, callback) { + this.gridfs.collection(this.bucketName + '.files').findOneAndUpdate({ + _id: id + }, { + $inc: { + 'metadata.c': -1, + 'metadata.m': -magic + } + }, { + returnOriginal: false + }, (err, result) => { + if (err) { + return callback(err); + } + + if (!result || !result.value) { + return callback(null, false); + } + + if (result.value.metadata.c === 0 && result.value.metadata.m === 0) { + return this.gridstore.delete(id, err => { + if (err) { + return callback(err); + } + callback(null, 1); + }); + } + + return callback(null, 0); + }); + } + + calculateHash(input, callback) { + let algo = 'sha256'; + + if (!cryptoAsync) { + setImmediate(() => callback(null, crypto.createHash(algo).update(input).digest('hex'))); + return; + } + + cryptoAsync.hash(algo, input, (err, hash) => { + if (err) { + return callback(err); + } + return callback(null, hash.toString('hex')); + }); + } +} + +module.exports = AttachmentStorage; diff --git a/lib/handlers/on-copy.js b/lib/handlers/on-copy.js index 6f0999fd..faab25b1 100644 --- a/lib/handlers/on-copy.js +++ b/lib/handlers/on-copy.js @@ -5,7 +5,7 @@ const db = require('../db'); const tools = require('../tools'); // COPY / UID COPY sequence mailbox -module.exports = server => (path, update, session, callback) => { +module.exports = (server, messageHandler) => (path, update, session, callback) => { server.logger.debug( { tnx: 'copy', @@ -16,6 +16,7 @@ module.exports = server => (path, update, session, callback) => { path, update.destination ); + db.database.collection('mailboxes').findOne({ user: session.user.id, path @@ -151,8 +152,9 @@ module.exports = server => (path, update, session, callback) => { copiedMessages++; copiedStorage += Number(message.size) || 0; - let attachments = Object.keys(message.map || {}).map(key => message.map[key]); - if (!attachments.length) { + let attachmentIds = Object.keys(message.mimetree.attachmentMap || {}).map(key => message.mimetree.attachmentMap[key]); + + if (!attachmentIds.length) { let entry = { command: 'EXISTS', uid: message.uid, @@ -165,20 +167,7 @@ module.exports = server => (path, update, session, callback) => { return server.notifier.addEntries(session.user.id, target.path, entry, processNext); } - // update attachments - db.gridfs.collection('attachments.files').updateMany({ - _id: { - $in: attachments - } - }, { - $inc: { - 'metadata.c': 1, - 'metadata.m': message.magic - } - }, { - multi: true, - w: 1 - }, err => { + messageHandler.attachmentStorage.updateMany(attachmentIds, 1, message.magic, err => { if (err) { // should we care about this error? } diff --git a/lib/handlers/on-expunge.js b/lib/handlers/on-expunge.js index 98e05f04..99edc6ba 100644 --- a/lib/handlers/on-expunge.js +++ b/lib/handlers/on-expunge.js @@ -3,7 +3,7 @@ const db = require('../db'); // EXPUNGE deletes all messages in selected mailbox marked with \Delete -module.exports = server => (path, update, session, callback) => { +module.exports = (server, messageHandler) => (path, update, session, callback) => { server.logger.debug( { tnx: 'expunge', @@ -35,7 +35,7 @@ module.exports = server => (path, update, session, callback) => { _id: true, uid: true, size: true, - map: true, + 'mimeTree.attachmentMap': true, magic: true, unseen: true }) @@ -92,9 +92,9 @@ module.exports = server => (path, update, session, callback) => { deletedMessages++; deletedStorage += Number(message.size) || 0; - let attachments = Object.keys(message.map || {}).map(key => message.map[key]); + let attachmentIds = Object.keys(message.mimeTree.attachmentMap || {}).map(key => message.mimeTree.attachmentMap[key]); - if (!attachments.length) { + if (!attachmentIds.length) { // not stored attachments return server.notifier.addEntries( session.user.id, @@ -110,22 +110,9 @@ module.exports = server => (path, update, session, callback) => { ); } - // remove references to attachments (if any exist) - db.gridfs.collection('attachments.files').updateMany({ - _id: { - $in: attachments - } - }, { - $inc: { - 'metadata.c': -1, - 'metadata.m': -message.magic - } - }, { - multi: true, - w: 1 - }, err => { + messageHandler.attachmentStorage.updateMany(attachmentIds, -1, -message.magic, err => { if (err) { - // ignore as we don't really care if we have orphans or not + // should we care about this error? } server.notifier.addEntries( session.user.id, diff --git a/lib/handlers/on-fetch.js b/lib/handlers/on-fetch.js index 1e8f48be..0f872d13 100644 --- a/lib/handlers/on-fetch.js +++ b/lib/handlers/on-fetch.js @@ -7,7 +7,7 @@ const db = require('../db'); const tools = require('../tools'); const consts = require('../consts'); -module.exports = server => (path, options, session, callback) => { +module.exports = (server, messageHandler) => (path, options, session, callback) => { server.logger.debug( { tnx: 'fetch', @@ -119,7 +119,7 @@ module.exports = server => (path, options, session, callback) => { logger: server.logger, fetchOptions: {}, database: db.database, - gridfs: db.gridfs, + attachmentStorage: messageHandler.attachmentStorage, acceptUTF8Enabled: session.isUTF8Enabled() }) }) diff --git a/lib/message-handler.js b/lib/message-handler.js index 7a27d225..84df5f41 100644 --- a/lib/message-handler.js +++ b/lib/message-handler.js @@ -5,6 +5,7 @@ const uuidV1 = require('uuid/v1'); const ObjectID = require('mongodb').ObjectID; const Indexer = require('../imap-core/lib/indexer/indexer'); const ImapNotifier = require('./imap-notifier'); +const AttachmentStorage = require('./attachment-storage'); const libmime = require('libmime'); const counters = require('./counters'); const consts = require('./consts'); @@ -21,17 +22,24 @@ class MessageHandler { constructor(options) { this.database = options.database; this.redis = options.redis; + + this.attachmentStorage = + options.attachmentStorage || + new AttachmentStorage({ + gridfs: options.gridfs || options.database + }); + this.indexer = new Indexer({ - database: options.database, - gridfs: options.gridfs + attachmentStorage: this.attachmentStorage }); + this.notifier = new ImapNotifier({ database: options.database, redis: this.redis, pushOnly: true }); + this.users = options.users || options.database; - this.gridfs = options.gridfs || options.database; this.counters = counters(this.redis); } @@ -111,17 +119,12 @@ class MessageHandler { return callback(...args); } - let attachments = Object.keys(maildata.map || {}).map(key => maildata.map[key]); - if (!attachments.length) { + let attachmentIds = Object.keys(mimeTree.attachmentMap || {}).map(key => mimeTree.attachmentMap[key]); + if (!attachmentIds.length) { return callback(...args); } - // error occured, remove attachments - this.gridfs.collection('attachments.files').deleteMany({ - _id: { - $in: attachments - } - }, () => callback(...args)); + this.attachmentStorage.deleteMany(attachmentIds, maildata.magic, () => callback(...args)); }; this.indexer.storeNodeBodies(id, maildata, mimeTree, err => { @@ -130,7 +133,7 @@ class MessageHandler { } // prepare message object - let message = { + let messageData = { _id: id, // should be kept when COPY'ing or MOVE'ing @@ -166,37 +169,38 @@ class MessageHandler { draft: flags.includes('\\Draft'), magic: maildata.magic, - map: maildata.map, subject }; if (maildata.attachments && maildata.attachments.length) { - message.attachments = maildata.attachments; - message.ha = true; + messageData.attachments = maildata.attachments; + messageData.ha = true; } else { - message.ha = false; + messageData.ha = false; } if (maildata.text) { - message.text = maildata.text.replace(/\r\n/g, '\n').trim(); + messageData.text = maildata.text.replace(/\r\n/g, '\n').trim(); // text is indexed with a fulltext index, so only store the beginning of it - message.text = - message.text.length <= consts.MAX_PLAINTEXT_CONTENT ? message.text : message.text.substr(0, consts.MAX_PLAINTEXT_CONTENT); - message.intro = message.text.replace(/\s+/g, ' ').trim(); - if (message.intro.length > 128) { - let intro = message.intro.substr(0, 128); + messageData.text = + messageData.text.length <= consts.MAX_PLAINTEXT_CONTENT + ? messageData.text + : messageData.text.substr(0, consts.MAX_PLAINTEXT_CONTENT); + messageData.intro = messageData.text.replace(/\s+/g, ' ').trim(); + if (messageData.intro.length > 128) { + let intro = messageData.intro.substr(0, 128); let lastSp = intro.lastIndexOf(' '); if (lastSp > 0) { intro = intro.substr(0, lastSp); } - message.intro = intro + '…'; + messageData.intro = intro + '…'; } } if (maildata.html && maildata.html.length) { let htmlSize = 0; - message.html = maildata.html + messageData.html = maildata.html .map(html => { if (htmlSize >= consts.MAX_HTML_CONTENT || !html) { return ''; @@ -262,17 +266,17 @@ class MessageHandler { let mailboxData = item.value; // updated message object by setting mailbox specific values - message.mailbox = mailboxData._id; - message.user = mailboxData.user; - message.uid = mailboxData.uidNext; - message.modseq = mailboxData.modifyIndex + 1; + messageData.mailbox = mailboxData._id; + messageData.user = mailboxData.user; + messageData.uid = mailboxData.uidNext; + messageData.modseq = mailboxData.modifyIndex + 1; if (!['\\Junk', '\\Trash'].includes(mailboxData.specialUse)) { - message.searchable = true; + messageData.searchable = true; } if (mailboxData.specialUse === '\\Junk') { - message.junk = true; + messageData.junk = true; } this.getThreadId(mailboxData.user, subject, mimeTree, (err, thread) => { @@ -280,18 +284,18 @@ class MessageHandler { return rollback(err); } - message.thread = thread; + messageData.thread = thread; - this.database.collection('messages').insertOne(message, err => { + this.database.collection('messages').insertOne(messageData, err => { if (err) { return rollback(err); } let uidValidity = mailboxData.uidValidity; - let uid = message.uid; + let uid = messageData.uid; if (options.session && options.session.selected && options.session.selected.mailbox === mailboxData.path) { - options.session.writeStream.write(options.session.formatResponse('EXISTS', message.uid)); + options.session.writeStream.write(options.session.formatResponse('EXISTS', messageData.uid)); } this.notifier.addEntries( @@ -299,18 +303,18 @@ class MessageHandler { false, { command: 'EXISTS', - uid: message.uid, + uid: messageData.uid, ignore: options.session && options.session.id, - message: message._id, - modseq: message.modseq, - unseen: message.unseen + message: messageData._id, + modseq: messageData.modseq, + unseen: messageData.unseen }, () => { this.notifier.fire(mailboxData.user, mailboxData.path); return cleanup(null, true, { uidValidity, uid, - id: message._id, + id: messageData._id, status: 'new' }); } @@ -494,30 +498,12 @@ class MessageHandler { }, () => { let updateAttachments = next => { - let attachments = Object.keys(message.map || {}).map(key => message.map[key]); - if (!attachments.length) { + let attachmentIds = Object.keys(message.mimeTree.attachmentMap || {}).map(key => message.mimeTree.attachmentMap[key]); + if (!attachmentIds.length) { return next(); } - // remove link to message from attachments (if any exist) - this.gridfs.collection('attachments.files').updateMany({ - _id: { - $in: attachments - } - }, { - $inc: { - 'metadata.c': -1, - 'metadata.m': -message.magic - } - }, { - multi: true, - w: 1 - }, err => { - if (err) { - // ignore as we don't really care if we have orphans or not - } - next(); - }); + this.attachmentStorage.deleteMany(attachmentIds, next); }; updateAttachments(() => { diff --git a/lmtp.js b/lmtp.js index 175cb6c6..5f7e9ecb 100644 --- a/lmtp.js +++ b/lmtp.js @@ -432,7 +432,12 @@ module.exports = done => { return setImmediate(() => done(null, false)); } - messageHandler = new MessageHandler({ database: db.database, gridfs: db.gridfs, users: db.users, redis: db.redis }); + messageHandler = new MessageHandler({ + database: db.database, + gridfs: db.gridfs, + users: db.users, + redis: db.redis + }); let started = false; diff --git a/pop3.js b/pop3.js index dba32c9d..f7d760d5 100644 --- a/pop3.js +++ b/pop3.js @@ -307,8 +307,17 @@ module.exports = done => { let started = false; - messageHandler = new MessageHandler({ database: db.database, gridfs: db.gridfs, redis: db.redis }); - userHandler = new UserHandler({ database: db.database, users: db.users, redis: db.redis }); + messageHandler = new MessageHandler({ + database: db.database, + gridfs: db.gridfs, + redis: db.redis + }); + + userHandler = new UserHandler({ + database: db.database, + users: db.users, + redis: db.redis + }); server.on('error', err => { if (!started) {