mirror of
https://github.com/nodemailer/wildduck.git
synced 2024-09-20 15:26:03 +08:00
refactor attachment storage
This commit is contained in:
parent
463bc61aeb
commit
4138bf2a2f
3
api.js
3
api.js
|
@ -100,18 +100,21 @@ module.exports = done => {
|
|||
database: db.database,
|
||||
redis: db.redis
|
||||
});
|
||||
|
||||
messageHandler = new MessageHandler({
|
||||
database: db.database,
|
||||
users: db.users,
|
||||
gridfs: db.gridfs,
|
||||
redis: db.redis
|
||||
});
|
||||
|
||||
userHandler = new UserHandler({
|
||||
database: db.database,
|
||||
users: db.users,
|
||||
redis: db.redis,
|
||||
messageHandler
|
||||
});
|
||||
|
||||
mailboxHandler = new MailboxHandler({
|
||||
database: db.database,
|
||||
users: db.users,
|
||||
|
|
|
@ -6,8 +6,6 @@ const PassThrough = stream.PassThrough;
|
|||
const BodyStructure = require('./body-structure');
|
||||
const createEnvelope = require('./create-envelope');
|
||||
const parseMimeTree = require('./parse-mime-tree');
|
||||
const ObjectID = require('mongodb').ObjectID;
|
||||
const GridFSBucket = require('mongodb').GridFSBucket;
|
||||
const libmime = require('libmime');
|
||||
const libqp = require('libqp');
|
||||
const libbase64 = require('libbase64');
|
||||
|
@ -16,25 +14,12 @@ const he = require('he');
|
|||
const htmlToText = require('html-to-text');
|
||||
const crypto = require('crypto');
|
||||
|
||||
let cryptoAsync;
|
||||
try {
|
||||
cryptoAsync = require('@ronomon/crypto-async'); // eslint-disable-line global-require
|
||||
} catch (E) {
|
||||
// ignore
|
||||
}
|
||||
|
||||
class Indexer {
|
||||
constructor(options) {
|
||||
this.options = options || {};
|
||||
this.fetchOptions = this.options.fetchOptions || {};
|
||||
|
||||
this.database = this.options.database;
|
||||
this.gridfs = this.options.gridfs || this.options.database;
|
||||
if (this.gridfs) {
|
||||
this.gridstore = new GridFSBucket(this.gridfs, {
|
||||
bucketName: 'attachments'
|
||||
});
|
||||
}
|
||||
this.attachmentStorage = this.options.attachmentStorage;
|
||||
|
||||
// create logger
|
||||
this.logger = this.options.logger || {
|
||||
|
@ -195,7 +180,11 @@ class Indexer {
|
|||
} else if (node.attachmentId && !skipExternal) {
|
||||
append(false, true); // force newline between header and contents
|
||||
|
||||
let attachmentStream = this.gridstore.openDownloadStream(node.attachmentId);
|
||||
let attachmentId = node.attachmentId;
|
||||
if (mimeTree.attachmentMap && mimeTree.attachmentMap[node.attachmentId]) {
|
||||
attachmentId = mimeTree.attachmentMap[node.attachmentId];
|
||||
}
|
||||
let attachmentStream = this.attachmentStorage.createReadStream(attachmentId);
|
||||
|
||||
attachmentStream.once('error', err => {
|
||||
res.emit('error', err);
|
||||
|
@ -267,16 +256,13 @@ class Indexer {
|
|||
*/
|
||||
getMaildata(messageId, mimeTree) {
|
||||
let magic = parseInt(crypto.randomBytes(2).toString('hex'), 16);
|
||||
let map = {};
|
||||
let maildata = {
|
||||
nodes: [],
|
||||
attachments: [],
|
||||
text: '',
|
||||
html: [],
|
||||
// magic number to append to increment stored attachment object counter
|
||||
magic,
|
||||
// match ids referenced in document to actual attachment ids
|
||||
map
|
||||
magic
|
||||
};
|
||||
|
||||
let idcount = 0;
|
||||
|
@ -363,7 +349,6 @@ class Indexer {
|
|||
// remove attachments and very large text nodes from the mime tree
|
||||
if (!isMultipart && node.body && node.body.length && (!isInlineText || node.size > 300 * 1024)) {
|
||||
let attachmentId = 'ATT' + leftPad(++idcount, '0', 5);
|
||||
map[attachmentId] = new ObjectID();
|
||||
|
||||
let fileName =
|
||||
(node.parsedHeader['content-disposition'] &&
|
||||
|
@ -371,6 +356,7 @@ class Indexer {
|
|||
node.parsedHeader['content-disposition'].params.filename) ||
|
||||
(node.parsedHeader['content-type'] && node.parsedHeader['content-type'].params && node.parsedHeader['content-type'].params.name) ||
|
||||
false;
|
||||
|
||||
let contentId = (node.parsedHeader['content-id'] || '').toString().replace(/<|>/g, '').trim();
|
||||
|
||||
if (fileName) {
|
||||
|
@ -391,21 +377,9 @@ class Indexer {
|
|||
// push to queue
|
||||
maildata.nodes.push({
|
||||
attachmentId,
|
||||
options: {
|
||||
fsync: true,
|
||||
magic: maildata.magic,
|
||||
contentType,
|
||||
// metadata should include only minimally required information, this would allow
|
||||
// to share attachments between different messages if the content is exactly the same
|
||||
// even though metadata (filename, content-disposition etc) might not
|
||||
metadata: {
|
||||
// values to detect if there are messages that reference to this attachment or not
|
||||
m: maildata.magic,
|
||||
c: 1,
|
||||
|
||||
// how to decode contents if a webclient or API asks for the attachment
|
||||
transferEncoding
|
||||
}
|
||||
},
|
||||
transferEncoding,
|
||||
body: node.body
|
||||
});
|
||||
|
||||
|
@ -463,82 +437,19 @@ class Indexer {
|
|||
storeNodeBodies(messageId, maildata, mimeTree, callback) {
|
||||
let pos = 0;
|
||||
let nodes = maildata.nodes;
|
||||
mimeTree.attachmentMap = {};
|
||||
let storeNode = () => {
|
||||
if (pos >= nodes.length) {
|
||||
// replace attachment IDs with ObjectIDs in the mimeTree
|
||||
let walk = (node, next) => {
|
||||
if (node.attachmentId && maildata.map[node.attachmentId]) {
|
||||
node.attachmentId = maildata.map[node.attachmentId];
|
||||
}
|
||||
|
||||
if (Array.isArray(node.childNodes)) {
|
||||
let pos = 0;
|
||||
let processChildNodes = () => {
|
||||
if (pos >= node.childNodes.length) {
|
||||
return next();
|
||||
}
|
||||
let childNode = node.childNodes[pos++];
|
||||
walk(childNode, () => processChildNodes());
|
||||
};
|
||||
processChildNodes();
|
||||
} else {
|
||||
next();
|
||||
}
|
||||
};
|
||||
|
||||
return walk(mimeTree, () => callback(null, true));
|
||||
return callback(null, true);
|
||||
}
|
||||
|
||||
let node = nodes[pos++];
|
||||
|
||||
calculateHash(node.body, (err, hash) => {
|
||||
this.attachmentStorage.create(node, (err, id) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
this.gridfs.collection('attachments.files').findOneAndUpdate({
|
||||
'metadata.h': hash
|
||||
}, {
|
||||
$inc: {
|
||||
'metadata.c': 1,
|
||||
'metadata.m': maildata.magic
|
||||
}
|
||||
}, {
|
||||
returnOriginal: false
|
||||
}, (err, result) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
if (result && result.value) {
|
||||
maildata.map[node.attachmentId] = result.value._id;
|
||||
mimeTree.attachmentMap[node.attachmentId] = id;
|
||||
return storeNode();
|
||||
}
|
||||
|
||||
let returned = false;
|
||||
|
||||
node.options.metadata.h = hash;
|
||||
|
||||
let store = this.gridstore.openUploadStreamWithId(maildata.map[node.attachmentId], null, node.options);
|
||||
|
||||
store.once('error', err => {
|
||||
if (returned) {
|
||||
return;
|
||||
}
|
||||
returned = true;
|
||||
callback(err);
|
||||
});
|
||||
|
||||
store.once('finish', () => {
|
||||
if (returned) {
|
||||
return;
|
||||
}
|
||||
returned = true;
|
||||
return storeNode();
|
||||
});
|
||||
|
||||
store.end(node.body);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
|
@ -800,20 +711,4 @@ function leftPad(val, chr, len) {
|
|||
return chr.repeat(len - val.toString().length) + val;
|
||||
}
|
||||
|
||||
function calculateHash(input, callback) {
|
||||
let algo = 'sha256';
|
||||
|
||||
if (!cryptoAsync) {
|
||||
setImmediate(() => callback(null, crypto.createHash(algo).update(input).digest('hex')));
|
||||
return;
|
||||
}
|
||||
|
||||
cryptoAsync.hash(algo, input, (err, hash) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
return callback(null, hash.toString('hex'));
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = Indexer;
|
||||
|
|
29
imap.js
29
imap.js
|
@ -199,7 +199,7 @@ function clearExpiredMessages() {
|
|||
mailbox: true,
|
||||
uid: true,
|
||||
size: true,
|
||||
map: true,
|
||||
'mimeTree.attachmentMap': true,
|
||||
magic: true,
|
||||
unseen: true
|
||||
});
|
||||
|
@ -295,9 +295,24 @@ module.exports = done => {
|
|||
redis: db.redis
|
||||
});
|
||||
|
||||
messageHandler = new MessageHandler({ database: db.database, gridfs: db.gridfs, redis: db.redis });
|
||||
userHandler = new UserHandler({ database: db.database, users: db.users, redis: db.redis });
|
||||
mailboxHandler = new MailboxHandler({ database: db.database, users: db.users, redis: db.redis, notifier: server.notifier });
|
||||
messageHandler = new MessageHandler({
|
||||
database: db.database,
|
||||
gridfs: db.gridfs,
|
||||
redis: db.redis
|
||||
});
|
||||
|
||||
userHandler = new UserHandler({
|
||||
database: db.database,
|
||||
users: db.users,
|
||||
redis: db.redis
|
||||
});
|
||||
|
||||
mailboxHandler = new MailboxHandler({
|
||||
database: db.database,
|
||||
users: db.users,
|
||||
redis: db.redis,
|
||||
notifier: server.notifier
|
||||
});
|
||||
|
||||
let started = false;
|
||||
|
||||
|
@ -324,7 +339,7 @@ module.exports = done => {
|
|||
});
|
||||
|
||||
// setup command handlers for the server instance
|
||||
server.onFetch = onFetch(server);
|
||||
server.onFetch = onFetch(server, messageHandler);
|
||||
server.onAuth = onAuth(server, userHandler);
|
||||
server.onList = onList(server);
|
||||
server.onLsub = onLsub(server);
|
||||
|
@ -337,8 +352,8 @@ module.exports = done => {
|
|||
server.onStatus = onStatus(server);
|
||||
server.onAppend = onAppend(server, messageHandler);
|
||||
server.onStore = onStore(server);
|
||||
server.onExpunge = onExpunge(server);
|
||||
server.onCopy = onCopy(server);
|
||||
server.onExpunge = onExpunge(server, messageHandler);
|
||||
server.onCopy = onCopy(server, messageHandler);
|
||||
server.onMove = onMove(server, messageHandler);
|
||||
server.onSearch = onSearch(server);
|
||||
server.onGetQuotaRoot = onGetQuotaRoot(server);
|
||||
|
|
|
@ -5,7 +5,6 @@ const MongoPaging = require('mongo-cursor-pagination');
|
|||
const addressparser = require('addressparser');
|
||||
const ObjectID = require('mongodb').ObjectID;
|
||||
const tools = require('../tools');
|
||||
const GridFSBucket = require('mongodb').GridFSBucket;
|
||||
const libbase64 = require('libbase64');
|
||||
const libqp = require('libqp');
|
||||
|
||||
|
@ -667,7 +666,7 @@ module.exports = (db, server, messageHandler) => {
|
|||
_id: true,
|
||||
user: true,
|
||||
attachments: true,
|
||||
map: true
|
||||
'mimeTree.attachmentMap': true
|
||||
}
|
||||
}, (err, messageData) => {
|
||||
if (err) {
|
||||
|
@ -683,7 +682,7 @@ module.exports = (db, server, messageHandler) => {
|
|||
return next();
|
||||
}
|
||||
|
||||
let attachmentId = messageData.map[attachment];
|
||||
let attachmentId = messageData.mimeTree.attachmentMap && messageData.mimeTree.attachmentMap[attachment];
|
||||
if (!attachmentId) {
|
||||
res.json({
|
||||
error: 'This attachment does not exist'
|
||||
|
@ -691,36 +690,25 @@ module.exports = (db, server, messageHandler) => {
|
|||
return next();
|
||||
}
|
||||
|
||||
db.gridfs.collection('attachments.files').findOne({
|
||||
_id: attachmentId
|
||||
}, (err, attachmentData) => {
|
||||
messageHandler.attachmentStorage.get(attachmentId, (err, attachmentData) => {
|
||||
if (err) {
|
||||
res.json({
|
||||
error: err.message
|
||||
});
|
||||
return next();
|
||||
}
|
||||
if (!attachmentData) {
|
||||
res.json({
|
||||
error: 'This attachment does not exist'
|
||||
});
|
||||
return next();
|
||||
}
|
||||
|
||||
res.writeHead(200, {
|
||||
'Content-Type': attachmentData.contentType || 'application/octet-stream'
|
||||
});
|
||||
|
||||
let bucket = new GridFSBucket(db.gridfs, {
|
||||
bucketName: 'attachments'
|
||||
});
|
||||
let attachmentStream = bucket.openDownloadStream(attachmentId);
|
||||
let attachmentStream = messageHandler.attachmentStorage.createReadStream(attachmentId);
|
||||
|
||||
attachmentStream.once('error', err => res.emit('error', err));
|
||||
|
||||
if (attachmentData.metadata.transferEncoding === 'base64') {
|
||||
if (attachmentData.transferEncoding === 'base64') {
|
||||
attachmentStream.pipe(new libbase64.Decoder()).pipe(res);
|
||||
} else if (attachmentData.metadata.transferEncoding === 'quoted-printable') {
|
||||
} else if (attachmentData.transferEncoding === 'quoted-printable') {
|
||||
attachmentStream.pipe(new libqp.Decoder()).pipe(res);
|
||||
} else {
|
||||
attachmentStream.pipe(res);
|
||||
|
@ -876,7 +864,7 @@ module.exports = (db, server, messageHandler) => {
|
|||
mailbox: true,
|
||||
uid: true,
|
||||
size: true,
|
||||
map: true,
|
||||
'mimeTree.attachmentMap': true,
|
||||
magic: true,
|
||||
unseen: true
|
||||
}
|
||||
|
|
193
lib/attachment-storage.js
Normal file
193
lib/attachment-storage.js
Normal file
|
@ -0,0 +1,193 @@
|
|||
'use strict';
|
||||
|
||||
const ObjectID = require('mongodb').ObjectID;
|
||||
const crypto = require('crypto');
|
||||
const GridFSBucket = require('mongodb').GridFSBucket;
|
||||
let cryptoAsync;
|
||||
try {
|
||||
cryptoAsync = require('@ronomon/crypto-async'); // eslint-disable-line global-require
|
||||
} catch (E) {
|
||||
// ignore
|
||||
}
|
||||
|
||||
class AttachmentStorage {
|
||||
constructor(options) {
|
||||
this.bucketName = options.bucket || 'attachments';
|
||||
this.gridfs = options.gridfs;
|
||||
this.gridstore = new GridFSBucket(this.gridfs, {
|
||||
bucketName: this.bucketName
|
||||
});
|
||||
}
|
||||
|
||||
get(attachmentId, callback) {
|
||||
this.gridfs.collection('attachments.files').findOne({
|
||||
_id: attachmentId
|
||||
}, (err, attachmentData) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
if (!attachmentData) {
|
||||
return callback(new Error('This attachment does not exist'));
|
||||
}
|
||||
|
||||
return callback(null, {
|
||||
contentType: attachmentData.contentType,
|
||||
transferEncoding: attachmentData.metadata.transferEncoding,
|
||||
metadata: attachmentData.metadata
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
create(attachment, callback) {
|
||||
this.calculateHash(attachment.body, (err, hash) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
this.gridfs.collection(this.bucketName + '.files').findOneAndUpdate({
|
||||
'metadata.h': hash
|
||||
}, {
|
||||
$inc: {
|
||||
'metadata.c': 1,
|
||||
'metadata.m': attachment.magic
|
||||
}
|
||||
}, {
|
||||
returnOriginal: false
|
||||
}, (err, result) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
if (result && result.value) {
|
||||
return callback(null, result.value._id);
|
||||
}
|
||||
|
||||
let returned = false;
|
||||
|
||||
let id = new ObjectID();
|
||||
let metadata = {
|
||||
h: hash,
|
||||
m: attachment.magic,
|
||||
c: 1,
|
||||
transferEncoding: attachment.transferEncoding
|
||||
};
|
||||
Object.keys(attachment.metadata || {}).forEach(key => {
|
||||
if (!(key in attachment.metadata)) {
|
||||
metadata[key] = attachment.metadata[key];
|
||||
}
|
||||
});
|
||||
|
||||
let store = this.gridstore.openUploadStreamWithId(id, null, {
|
||||
contentType: attachment.contentType,
|
||||
metadata
|
||||
});
|
||||
|
||||
store.once('error', err => {
|
||||
if (returned) {
|
||||
return;
|
||||
}
|
||||
returned = true;
|
||||
callback(err);
|
||||
});
|
||||
|
||||
store.once('finish', () => {
|
||||
if (returned) {
|
||||
return;
|
||||
}
|
||||
returned = true;
|
||||
return callback(null, id);
|
||||
});
|
||||
|
||||
store.end(attachment.body);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
createReadStream(id) {
|
||||
return this.gridstore.openDownloadStream(id);
|
||||
}
|
||||
|
||||
deleteMany(ids, magic, callback) {
|
||||
let pos = 0;
|
||||
let deleteNext = () => {
|
||||
if (pos >= ids.length) {
|
||||
return callback(null, true);
|
||||
}
|
||||
let id = ids[pos++];
|
||||
this.delete(id, magic, deleteNext);
|
||||
};
|
||||
deleteNext();
|
||||
}
|
||||
|
||||
updateMany(ids, count, magic, callback) {
|
||||
// update attachments
|
||||
this.gridfs.collection(this.bucketName + '.files').updateMany(
|
||||
{
|
||||
_id: {
|
||||
$in: ids
|
||||
}
|
||||
},
|
||||
{
|
||||
$inc: {
|
||||
'metadata.c': count,
|
||||
'metadata.m': magic
|
||||
}
|
||||
},
|
||||
{
|
||||
multi: true,
|
||||
w: 1
|
||||
},
|
||||
callback
|
||||
);
|
||||
}
|
||||
|
||||
delete(id, magic, callback) {
|
||||
this.gridfs.collection(this.bucketName + '.files').findOneAndUpdate({
|
||||
_id: id
|
||||
}, {
|
||||
$inc: {
|
||||
'metadata.c': -1,
|
||||
'metadata.m': -magic
|
||||
}
|
||||
}, {
|
||||
returnOriginal: false
|
||||
}, (err, result) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
if (!result || !result.value) {
|
||||
return callback(null, false);
|
||||
}
|
||||
|
||||
if (result.value.metadata.c === 0 && result.value.metadata.m === 0) {
|
||||
return this.gridstore.delete(id, err => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
callback(null, 1);
|
||||
});
|
||||
}
|
||||
|
||||
return callback(null, 0);
|
||||
});
|
||||
}
|
||||
|
||||
calculateHash(input, callback) {
|
||||
let algo = 'sha256';
|
||||
|
||||
if (!cryptoAsync) {
|
||||
setImmediate(() => callback(null, crypto.createHash(algo).update(input).digest('hex')));
|
||||
return;
|
||||
}
|
||||
|
||||
cryptoAsync.hash(algo, input, (err, hash) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
return callback(null, hash.toString('hex'));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = AttachmentStorage;
|
|
@ -5,7 +5,7 @@ const db = require('../db');
|
|||
const tools = require('../tools');
|
||||
|
||||
// COPY / UID COPY sequence mailbox
|
||||
module.exports = server => (path, update, session, callback) => {
|
||||
module.exports = (server, messageHandler) => (path, update, session, callback) => {
|
||||
server.logger.debug(
|
||||
{
|
||||
tnx: 'copy',
|
||||
|
@ -16,6 +16,7 @@ module.exports = server => (path, update, session, callback) => {
|
|||
path,
|
||||
update.destination
|
||||
);
|
||||
|
||||
db.database.collection('mailboxes').findOne({
|
||||
user: session.user.id,
|
||||
path
|
||||
|
@ -151,8 +152,9 @@ module.exports = server => (path, update, session, callback) => {
|
|||
copiedMessages++;
|
||||
copiedStorage += Number(message.size) || 0;
|
||||
|
||||
let attachments = Object.keys(message.map || {}).map(key => message.map[key]);
|
||||
if (!attachments.length) {
|
||||
let attachmentIds = Object.keys(message.mimetree.attachmentMap || {}).map(key => message.mimetree.attachmentMap[key]);
|
||||
|
||||
if (!attachmentIds.length) {
|
||||
let entry = {
|
||||
command: 'EXISTS',
|
||||
uid: message.uid,
|
||||
|
@ -165,20 +167,7 @@ module.exports = server => (path, update, session, callback) => {
|
|||
return server.notifier.addEntries(session.user.id, target.path, entry, processNext);
|
||||
}
|
||||
|
||||
// update attachments
|
||||
db.gridfs.collection('attachments.files').updateMany({
|
||||
_id: {
|
||||
$in: attachments
|
||||
}
|
||||
}, {
|
||||
$inc: {
|
||||
'metadata.c': 1,
|
||||
'metadata.m': message.magic
|
||||
}
|
||||
}, {
|
||||
multi: true,
|
||||
w: 1
|
||||
}, err => {
|
||||
messageHandler.attachmentStorage.updateMany(attachmentIds, 1, message.magic, err => {
|
||||
if (err) {
|
||||
// should we care about this error?
|
||||
}
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
const db = require('../db');
|
||||
|
||||
// EXPUNGE deletes all messages in selected mailbox marked with \Delete
|
||||
module.exports = server => (path, update, session, callback) => {
|
||||
module.exports = (server, messageHandler) => (path, update, session, callback) => {
|
||||
server.logger.debug(
|
||||
{
|
||||
tnx: 'expunge',
|
||||
|
@ -35,7 +35,7 @@ module.exports = server => (path, update, session, callback) => {
|
|||
_id: true,
|
||||
uid: true,
|
||||
size: true,
|
||||
map: true,
|
||||
'mimeTree.attachmentMap': true,
|
||||
magic: true,
|
||||
unseen: true
|
||||
})
|
||||
|
@ -92,9 +92,9 @@ module.exports = server => (path, update, session, callback) => {
|
|||
deletedMessages++;
|
||||
deletedStorage += Number(message.size) || 0;
|
||||
|
||||
let attachments = Object.keys(message.map || {}).map(key => message.map[key]);
|
||||
let attachmentIds = Object.keys(message.mimeTree.attachmentMap || {}).map(key => message.mimeTree.attachmentMap[key]);
|
||||
|
||||
if (!attachments.length) {
|
||||
if (!attachmentIds.length) {
|
||||
// not stored attachments
|
||||
return server.notifier.addEntries(
|
||||
session.user.id,
|
||||
|
@ -110,22 +110,9 @@ module.exports = server => (path, update, session, callback) => {
|
|||
);
|
||||
}
|
||||
|
||||
// remove references to attachments (if any exist)
|
||||
db.gridfs.collection('attachments.files').updateMany({
|
||||
_id: {
|
||||
$in: attachments
|
||||
}
|
||||
}, {
|
||||
$inc: {
|
||||
'metadata.c': -1,
|
||||
'metadata.m': -message.magic
|
||||
}
|
||||
}, {
|
||||
multi: true,
|
||||
w: 1
|
||||
}, err => {
|
||||
messageHandler.attachmentStorage.updateMany(attachmentIds, -1, -message.magic, err => {
|
||||
if (err) {
|
||||
// ignore as we don't really care if we have orphans or not
|
||||
// should we care about this error?
|
||||
}
|
||||
server.notifier.addEntries(
|
||||
session.user.id,
|
||||
|
|
|
@ -7,7 +7,7 @@ const db = require('../db');
|
|||
const tools = require('../tools');
|
||||
const consts = require('../consts');
|
||||
|
||||
module.exports = server => (path, options, session, callback) => {
|
||||
module.exports = (server, messageHandler) => (path, options, session, callback) => {
|
||||
server.logger.debug(
|
||||
{
|
||||
tnx: 'fetch',
|
||||
|
@ -119,7 +119,7 @@ module.exports = server => (path, options, session, callback) => {
|
|||
logger: server.logger,
|
||||
fetchOptions: {},
|
||||
database: db.database,
|
||||
gridfs: db.gridfs,
|
||||
attachmentStorage: messageHandler.attachmentStorage,
|
||||
acceptUTF8Enabled: session.isUTF8Enabled()
|
||||
})
|
||||
})
|
||||
|
|
|
@ -5,6 +5,7 @@ const uuidV1 = require('uuid/v1');
|
|||
const ObjectID = require('mongodb').ObjectID;
|
||||
const Indexer = require('../imap-core/lib/indexer/indexer');
|
||||
const ImapNotifier = require('./imap-notifier');
|
||||
const AttachmentStorage = require('./attachment-storage');
|
||||
const libmime = require('libmime');
|
||||
const counters = require('./counters');
|
||||
const consts = require('./consts');
|
||||
|
@ -21,17 +22,24 @@ class MessageHandler {
|
|||
constructor(options) {
|
||||
this.database = options.database;
|
||||
this.redis = options.redis;
|
||||
this.indexer = new Indexer({
|
||||
database: options.database,
|
||||
gridfs: options.gridfs
|
||||
|
||||
this.attachmentStorage =
|
||||
options.attachmentStorage ||
|
||||
new AttachmentStorage({
|
||||
gridfs: options.gridfs || options.database
|
||||
});
|
||||
|
||||
this.indexer = new Indexer({
|
||||
attachmentStorage: this.attachmentStorage
|
||||
});
|
||||
|
||||
this.notifier = new ImapNotifier({
|
||||
database: options.database,
|
||||
redis: this.redis,
|
||||
pushOnly: true
|
||||
});
|
||||
|
||||
this.users = options.users || options.database;
|
||||
this.gridfs = options.gridfs || options.database;
|
||||
this.counters = counters(this.redis);
|
||||
}
|
||||
|
||||
|
@ -111,17 +119,12 @@ class MessageHandler {
|
|||
return callback(...args);
|
||||
}
|
||||
|
||||
let attachments = Object.keys(maildata.map || {}).map(key => maildata.map[key]);
|
||||
if (!attachments.length) {
|
||||
let attachmentIds = Object.keys(mimeTree.attachmentMap || {}).map(key => mimeTree.attachmentMap[key]);
|
||||
if (!attachmentIds.length) {
|
||||
return callback(...args);
|
||||
}
|
||||
|
||||
// error occured, remove attachments
|
||||
this.gridfs.collection('attachments.files').deleteMany({
|
||||
_id: {
|
||||
$in: attachments
|
||||
}
|
||||
}, () => callback(...args));
|
||||
this.attachmentStorage.deleteMany(attachmentIds, maildata.magic, () => callback(...args));
|
||||
};
|
||||
|
||||
this.indexer.storeNodeBodies(id, maildata, mimeTree, err => {
|
||||
|
@ -130,7 +133,7 @@ class MessageHandler {
|
|||
}
|
||||
|
||||
// prepare message object
|
||||
let message = {
|
||||
let messageData = {
|
||||
_id: id,
|
||||
|
||||
// should be kept when COPY'ing or MOVE'ing
|
||||
|
@ -166,37 +169,38 @@ class MessageHandler {
|
|||
draft: flags.includes('\\Draft'),
|
||||
|
||||
magic: maildata.magic,
|
||||
map: maildata.map,
|
||||
|
||||
subject
|
||||
};
|
||||
|
||||
if (maildata.attachments && maildata.attachments.length) {
|
||||
message.attachments = maildata.attachments;
|
||||
message.ha = true;
|
||||
messageData.attachments = maildata.attachments;
|
||||
messageData.ha = true;
|
||||
} else {
|
||||
message.ha = false;
|
||||
messageData.ha = false;
|
||||
}
|
||||
|
||||
if (maildata.text) {
|
||||
message.text = maildata.text.replace(/\r\n/g, '\n').trim();
|
||||
messageData.text = maildata.text.replace(/\r\n/g, '\n').trim();
|
||||
// text is indexed with a fulltext index, so only store the beginning of it
|
||||
message.text =
|
||||
message.text.length <= consts.MAX_PLAINTEXT_CONTENT ? message.text : message.text.substr(0, consts.MAX_PLAINTEXT_CONTENT);
|
||||
message.intro = message.text.replace(/\s+/g, ' ').trim();
|
||||
if (message.intro.length > 128) {
|
||||
let intro = message.intro.substr(0, 128);
|
||||
messageData.text =
|
||||
messageData.text.length <= consts.MAX_PLAINTEXT_CONTENT
|
||||
? messageData.text
|
||||
: messageData.text.substr(0, consts.MAX_PLAINTEXT_CONTENT);
|
||||
messageData.intro = messageData.text.replace(/\s+/g, ' ').trim();
|
||||
if (messageData.intro.length > 128) {
|
||||
let intro = messageData.intro.substr(0, 128);
|
||||
let lastSp = intro.lastIndexOf(' ');
|
||||
if (lastSp > 0) {
|
||||
intro = intro.substr(0, lastSp);
|
||||
}
|
||||
message.intro = intro + '…';
|
||||
messageData.intro = intro + '…';
|
||||
}
|
||||
}
|
||||
|
||||
if (maildata.html && maildata.html.length) {
|
||||
let htmlSize = 0;
|
||||
message.html = maildata.html
|
||||
messageData.html = maildata.html
|
||||
.map(html => {
|
||||
if (htmlSize >= consts.MAX_HTML_CONTENT || !html) {
|
||||
return '';
|
||||
|
@ -262,17 +266,17 @@ class MessageHandler {
|
|||
let mailboxData = item.value;
|
||||
|
||||
// updated message object by setting mailbox specific values
|
||||
message.mailbox = mailboxData._id;
|
||||
message.user = mailboxData.user;
|
||||
message.uid = mailboxData.uidNext;
|
||||
message.modseq = mailboxData.modifyIndex + 1;
|
||||
messageData.mailbox = mailboxData._id;
|
||||
messageData.user = mailboxData.user;
|
||||
messageData.uid = mailboxData.uidNext;
|
||||
messageData.modseq = mailboxData.modifyIndex + 1;
|
||||
|
||||
if (!['\\Junk', '\\Trash'].includes(mailboxData.specialUse)) {
|
||||
message.searchable = true;
|
||||
messageData.searchable = true;
|
||||
}
|
||||
|
||||
if (mailboxData.specialUse === '\\Junk') {
|
||||
message.junk = true;
|
||||
messageData.junk = true;
|
||||
}
|
||||
|
||||
this.getThreadId(mailboxData.user, subject, mimeTree, (err, thread) => {
|
||||
|
@ -280,18 +284,18 @@ class MessageHandler {
|
|||
return rollback(err);
|
||||
}
|
||||
|
||||
message.thread = thread;
|
||||
messageData.thread = thread;
|
||||
|
||||
this.database.collection('messages').insertOne(message, err => {
|
||||
this.database.collection('messages').insertOne(messageData, err => {
|
||||
if (err) {
|
||||
return rollback(err);
|
||||
}
|
||||
|
||||
let uidValidity = mailboxData.uidValidity;
|
||||
let uid = message.uid;
|
||||
let uid = messageData.uid;
|
||||
|
||||
if (options.session && options.session.selected && options.session.selected.mailbox === mailboxData.path) {
|
||||
options.session.writeStream.write(options.session.formatResponse('EXISTS', message.uid));
|
||||
options.session.writeStream.write(options.session.formatResponse('EXISTS', messageData.uid));
|
||||
}
|
||||
|
||||
this.notifier.addEntries(
|
||||
|
@ -299,18 +303,18 @@ class MessageHandler {
|
|||
false,
|
||||
{
|
||||
command: 'EXISTS',
|
||||
uid: message.uid,
|
||||
uid: messageData.uid,
|
||||
ignore: options.session && options.session.id,
|
||||
message: message._id,
|
||||
modseq: message.modseq,
|
||||
unseen: message.unseen
|
||||
message: messageData._id,
|
||||
modseq: messageData.modseq,
|
||||
unseen: messageData.unseen
|
||||
},
|
||||
() => {
|
||||
this.notifier.fire(mailboxData.user, mailboxData.path);
|
||||
return cleanup(null, true, {
|
||||
uidValidity,
|
||||
uid,
|
||||
id: message._id,
|
||||
id: messageData._id,
|
||||
status: 'new'
|
||||
});
|
||||
}
|
||||
|
@ -494,30 +498,12 @@ class MessageHandler {
|
|||
},
|
||||
() => {
|
||||
let updateAttachments = next => {
|
||||
let attachments = Object.keys(message.map || {}).map(key => message.map[key]);
|
||||
if (!attachments.length) {
|
||||
let attachmentIds = Object.keys(message.mimeTree.attachmentMap || {}).map(key => message.mimeTree.attachmentMap[key]);
|
||||
if (!attachmentIds.length) {
|
||||
return next();
|
||||
}
|
||||
|
||||
// remove link to message from attachments (if any exist)
|
||||
this.gridfs.collection('attachments.files').updateMany({
|
||||
_id: {
|
||||
$in: attachments
|
||||
}
|
||||
}, {
|
||||
$inc: {
|
||||
'metadata.c': -1,
|
||||
'metadata.m': -message.magic
|
||||
}
|
||||
}, {
|
||||
multi: true,
|
||||
w: 1
|
||||
}, err => {
|
||||
if (err) {
|
||||
// ignore as we don't really care if we have orphans or not
|
||||
}
|
||||
next();
|
||||
});
|
||||
this.attachmentStorage.deleteMany(attachmentIds, next);
|
||||
};
|
||||
|
||||
updateAttachments(() => {
|
||||
|
|
7
lmtp.js
7
lmtp.js
|
@ -432,7 +432,12 @@ module.exports = done => {
|
|||
return setImmediate(() => done(null, false));
|
||||
}
|
||||
|
||||
messageHandler = new MessageHandler({ database: db.database, gridfs: db.gridfs, users: db.users, redis: db.redis });
|
||||
messageHandler = new MessageHandler({
|
||||
database: db.database,
|
||||
gridfs: db.gridfs,
|
||||
users: db.users,
|
||||
redis: db.redis
|
||||
});
|
||||
|
||||
let started = false;
|
||||
|
||||
|
|
13
pop3.js
13
pop3.js
|
@ -307,8 +307,17 @@ module.exports = done => {
|
|||
|
||||
let started = false;
|
||||
|
||||
messageHandler = new MessageHandler({ database: db.database, gridfs: db.gridfs, redis: db.redis });
|
||||
userHandler = new UserHandler({ database: db.database, users: db.users, redis: db.redis });
|
||||
messageHandler = new MessageHandler({
|
||||
database: db.database,
|
||||
gridfs: db.gridfs,
|
||||
redis: db.redis
|
||||
});
|
||||
|
||||
userHandler = new UserHandler({
|
||||
database: db.database,
|
||||
users: db.users,
|
||||
redis: db.redis
|
||||
});
|
||||
|
||||
server.on('error', err => {
|
||||
if (!started) {
|
||||
|
|
Loading…
Reference in a new issue